diff options
479 files changed, 15731 insertions, 6316 deletions
diff --git a/Documentation/ABI/testing/ppc-memtrace b/Documentation/ABI/testing/ppc-memtrace index 2e8b93741270..9606aed33137 100644 --- a/Documentation/ABI/testing/ppc-memtrace +++ b/Documentation/ABI/testing/ppc-memtrace @@ -13,10 +13,11 @@ Contact: linuxppc-dev@lists.ozlabs.org Description: Write an integer containing the size in bytes of the memory you want removed from each NUMA node to this file - it must be aligned to the memblock size. This amount of RAM will be removed - from the kernel mappings and the following debugfs files will be - created. This can only be successfully done once per boot. Once - memory is successfully removed from each node, the following - files are created. + from each NUMA node in the kernel mappings and the following + debugfs files will be created. Once memory is successfully + removed from each node, the following files are created. To + re-add memory to the kernel, echo 0 into this file (it will be + automatically onlined). What: /sys/kernel/debug/powerpc/memtrace/<node-id> Date: Aug 2017 diff --git a/Documentation/admin-guide/kernel-parameters.txt b/Documentation/admin-guide/kernel-parameters.txt index adafe47ac376..d60b169bb54b 100644 --- a/Documentation/admin-guide/kernel-parameters.txt +++ b/Documentation/admin-guide/kernel-parameters.txt @@ -2784,6 +2784,10 @@ nosmt=force: Force disable SMT, cannot be undone via the sysfs control file. + nospectre_v1 [PPC] Disable mitigations for Spectre Variant 1 (bounds + check bypass). With this option data leaks are possible + in the system. + nospectre_v2 [X86] Disable all mitigations for the Spectre variant 2 (indirect branch prediction) vulnerability. System may allow data leaks with this option, which is equivalent diff --git a/Documentation/device-mapper/delay.txt b/Documentation/device-mapper/delay.txt index 4b1d22a44ce4..6426c45273cb 100644 --- a/Documentation/device-mapper/delay.txt +++ b/Documentation/device-mapper/delay.txt @@ -5,7 +5,8 @@ Device-Mapper's "delay" target delays reads and/or writes and maps them to different devices. Parameters: - <device> <offset> <delay> [<write_device> <write_offset> <write_delay>] + <device> <offset> <delay> [<write_device> <write_offset> <write_delay> + [<flush_device> <flush_offset> <flush_delay>]] With separate write parameters, the first set is only used for reads. Offsets are specified in sectors. diff --git a/Documentation/device-mapper/dm-integrity.txt b/Documentation/device-mapper/dm-integrity.txt index f33e3ade7a09..297251b0d2d5 100644 --- a/Documentation/device-mapper/dm-integrity.txt +++ b/Documentation/device-mapper/dm-integrity.txt @@ -113,6 +113,10 @@ internal_hash:algorithm(:key) (the key is optional) from an upper layer target, such as dm-crypt. The upper layer target should check the validity of the integrity tags. +recalculate + Recalculate the integrity tags automatically. It is only valid + when using internal hash. + journal_crypt:algorithm(:key) (the key is optional) Encrypt the journal using given algorithm to make sure that the attacker can't read the journal. You can use a block cipher here diff --git a/Documentation/device-mapper/thin-provisioning.txt b/Documentation/device-mapper/thin-provisioning.txt index 3d01948ea061..883e7ca5f745 100644 --- a/Documentation/device-mapper/thin-provisioning.txt +++ b/Documentation/device-mapper/thin-provisioning.txt @@ -28,17 +28,18 @@ administrator some freedom, for example to: Status ====== -These targets are very much still in the EXPERIMENTAL state. Please -do not yet rely on them in production. But do experiment and offer us -feedback. Different use cases will have different performance -characteristics, for example due to fragmentation of the data volume. +These targets are considered safe for production use. But different use +cases will have different performance characteristics, for example due +to fragmentation of the data volume. If you find this software is not performing as expected please mail dm-devel@redhat.com with details and we'll try our best to improve things for you. -Userspace tools for checking and repairing the metadata are under -development. +Userspace tools for checking and repairing the metadata have been fully +developed and are available as 'thin_check' and 'thin_repair'. The name +of the package that provides these utilities varies by distribution (on +a Red Hat distribution it is named 'device-mapper-persistent-data'). Cookbook ======== @@ -280,7 +281,7 @@ ii) Status <transaction id> <used metadata blocks>/<total metadata blocks> <used data blocks>/<total data blocks> <held metadata root> ro|rw|out_of_data_space [no_]discard_passdown [error|queue]_if_no_space - needs_check|- + needs_check|- metadata_low_watermark transaction id: A 64-bit number used by userspace to help synchronise with metadata @@ -327,6 +328,11 @@ ii) Status thin-pool can be made fully operational again. '-' indicates needs_check is not set. + metadata_low_watermark: + Value of metadata low watermark in blocks. The kernel sets this + value internally but userspace needs to know this value to + determine if an event was caused by crossing this threshold. + iii) Messages create_thin <dev id> diff --git a/Documentation/hwmon/ibmpowernv b/Documentation/hwmon/ibmpowernv index 8826ba29db36..56468258711f 100644 --- a/Documentation/hwmon/ibmpowernv +++ b/Documentation/hwmon/ibmpowernv @@ -33,9 +33,48 @@ fanX_input Measured RPM value. fanX_min Threshold RPM for alert generation. fanX_fault 0: No fail condition 1: Failing fan + tempX_input Measured ambient temperature. tempX_max Threshold ambient temperature for alert generation. -inX_input Measured power supply voltage +tempX_highest Historical maximum temperature +tempX_lowest Historical minimum temperature +tempX_enable Enable/disable all temperature sensors belonging to the + sub-group. In POWER9, this attribute corresponds to + each OCC. Using this attribute each OCC can be asked to + disable/enable all of its temperature sensors. + 1: Enable + 0: Disable + +inX_input Measured power supply voltage (millivolt) inX_fault 0: No fail condition. 1: Failing power supply. -power1_input System power consumption (microWatt) +inX_highest Historical maximum voltage +inX_lowest Historical minimum voltage +inX_enable Enable/disable all voltage sensors belonging to the + sub-group. In POWER9, this attribute corresponds to + each OCC. Using this attribute each OCC can be asked to + disable/enable all of its voltage sensors. + 1: Enable + 0: Disable + +powerX_input Power consumption (microWatt) +powerX_input_highest Historical maximum power +powerX_input_lowest Historical minimum power +powerX_enable Enable/disable all power sensors belonging to the + sub-group. In POWER9, this attribute corresponds to + each OCC. Using this attribute each OCC can be asked to + disable/enable all of its power sensors. + 1: Enable + 0: Disable + +currX_input Measured current (milliampere) +currX_highest Historical maximum current +currX_lowest Historical minimum current +currX_enable Enable/disable all current sensors belonging to the + sub-group. In POWER9, this attribute corresponds to + each OCC. Using this attribute each OCC can be asked to + disable/enable all of its current sensors. + 1: Enable + 0: Disable + +energyX_input Cumulative energy (microJoule) diff --git a/Documentation/powerpc/DAWR-POWER9.txt b/Documentation/powerpc/DAWR-POWER9.txt new file mode 100644 index 000000000000..2feaa6619658 --- /dev/null +++ b/Documentation/powerpc/DAWR-POWER9.txt @@ -0,0 +1,58 @@ +DAWR issues on POWER9 +============================ + +On POWER9 the DAWR can cause a checkstop if it points to cache +inhibited (CI) memory. Currently Linux has no way to disinguish CI +memory when configuring the DAWR, so (for now) the DAWR is disabled by +this commit: + + commit 9654153158d3e0684a1bdb76dbababdb7111d5a0 + Author: Michael Neuling <mikey@neuling.org> + Date: Tue Mar 27 15:37:24 2018 +1100 + powerpc: Disable DAWR in the base POWER9 CPU features + +Technical Details: +============================ + +DAWR has 6 different ways of being set. +1) ptrace +2) h_set_mode(DAWR) +3) h_set_dabr() +4) kvmppc_set_one_reg() +5) xmon + +For ptrace, we now advertise zero breakpoints on POWER9 via the +PPC_PTRACE_GETHWDBGINFO call. This results in GDB falling back to +software emulation of the watchpoint (which is slow). + +h_set_mode(DAWR) and h_set_dabr() will now return an error to the +guest on a POWER9 host. Current Linux guests ignore this error, so +they will silently not get the DAWR. + +kvmppc_set_one_reg() will store the value in the vcpu but won't +actually set it on POWER9 hardware. This is done so we don't break +migration from POWER8 to POWER9, at the cost of silently losing the +DAWR on the migration. + +For xmon, the 'bd' command will return an error on P9. + +Consequences for users +============================ + +For GDB watchpoints (ie 'watch' command) on POWER9 bare metal , GDB +will accept the command. Unfortunately since there is no hardware +support for the watchpoint, GDB will software emulate the watchpoint +making it run very slowly. + +The same will also be true for any guests started on a POWER9 +host. The watchpoint will fail and GDB will fall back to software +emulation. + +If a guest is started on a POWER8 host, GDB will accept the watchpoint +and configure the hardware to use the DAWR. This will run at full +speed since it can use the hardware emulation. Unfortunately if this +guest is migrated to a POWER9 host, the watchpoint will be lost on the +POWER9. Loads and stores to the watchpoint locations will not be +trapped in GDB. The watchpoint is remembered, so if the guest is +migrated back to the POWER8 host, it will start working again. + diff --git a/Documentation/powerpc/transactional_memory.txt b/Documentation/powerpc/transactional_memory.txt index e32fdbb4c9a7..52c023e14f26 100644 --- a/Documentation/powerpc/transactional_memory.txt +++ b/Documentation/powerpc/transactional_memory.txt @@ -198,3 +198,47 @@ presented). The transaction cannot then be continued and will take the failure handler route. Furthermore, the transactional 2nd register state will be inaccessible. GDB can currently be used on programs using TM, but not sensibly in parts within transactions. + +POWER9 +====== + +TM on POWER9 has issues with storing the complete register state. This +is described in this commit: + + commit 4bb3c7a0208fc13ca70598efd109901a7cd45ae7 + Author: Paul Mackerras <paulus@ozlabs.org> + Date: Wed Mar 21 21:32:01 2018 +1100 + KVM: PPC: Book3S HV: Work around transactional memory bugs in POWER9 + +To account for this different POWER9 chips have TM enabled in +different ways. + +On POWER9N DD2.01 and below, TM is disabled. ie +HWCAP2[PPC_FEATURE2_HTM] is not set. + +On POWER9N DD2.1 TM is configured by firmware to always abort a +transaction when tm suspend occurs. So tsuspend will cause a +transaction to be aborted and rolled back. Kernel exceptions will also +cause the transaction to be aborted and rolled back and the exception +will not occur. If userspace constructs a sigcontext that enables TM +suspend, the sigcontext will be rejected by the kernel. This mode is +advertised to users with HWCAP2[PPC_FEATURE2_HTM_NO_SUSPEND] set. +HWCAP2[PPC_FEATURE2_HTM] is not set in this mode. + +On POWER9N DD2.2 and above, KVM and POWERVM emulate TM for guests (as +described in commit 4bb3c7a0208f), hence TM is enabled for guests +ie. HWCAP2[PPC_FEATURE2_HTM] is set for guest userspace. Guests that +makes heavy use of TM suspend (tsuspend or kernel suspend) will result +in traps into the hypervisor and hence will suffer a performance +degradation. Host userspace has TM disabled +ie. HWCAP2[PPC_FEATURE2_HTM] is not set. (although we make enable it +at some point in the future if we bring the emulation into host +userspace context switching). + +POWER9C DD1.2 and above are only available with POWERVM and hence +Linux only runs as a guest. On these systems TM is emulated like on +POWER9N DD2.2. + +Guest migration from POWER8 to POWER9 will work with POWER9N DD2.2 and +POWER9C DD1.2. Since earlier POWER9 processors don't support TM +emulation, migration from POWER8 to POWER9 is not supported there. diff --git a/MAINTAINERS b/MAINTAINERS index f72e524445a9..b6dcaaa23adf 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -813,11 +813,6 @@ S: Supported F: drivers/net/ethernet/amd/xgbe/ F: arch/arm64/boot/dts/amd/amd-seattle-xgbe*.dtsi -AMS (Apple Motion Sensor) DRIVER -M: Michael Hanselmann <linux-kernel@hansmi.ch> -S: Supported -F: drivers/macintosh/ams/ - ANALOG DEVICES INC AD5686 DRIVER M: Stefan Popa <stefan.popa@analog.com> L: linux-pm@vger.kernel.org diff --git a/arch/arm/include/asm/module.h b/arch/arm/include/asm/module.h index 89ad0596033a..9e81b7c498d8 100644 --- a/arch/arm/include/asm/module.h +++ b/arch/arm/include/asm/module.h @@ -34,6 +34,7 @@ struct mod_arch_specific { #endif }; +struct module; u32 get_module_plt(struct module *mod, unsigned long loc, Elf32_Addr val); /* diff --git a/arch/arm64/kernel/jump_label.c b/arch/arm64/kernel/jump_label.c index c2dd1ad3e648..e0756416e567 100644 --- a/arch/arm64/kernel/jump_label.c +++ b/arch/arm64/kernel/jump_label.c @@ -36,7 +36,7 @@ void arch_jump_label_transform(struct jump_entry *entry, insn = aarch64_insn_gen_nop(); } - aarch64_insn_patch_text(&addr, &insn, 1); + aarch64_insn_patch_text_nosync(addr, insn); } void arch_jump_label_transform_static(struct jump_entry *entry, diff --git a/arch/arm64/mm/init.c b/arch/arm64/mm/init.c index 9abf8a1e7b25..787e27964ab9 100644 --- a/arch/arm64/mm/init.c +++ b/arch/arm64/mm/init.c @@ -287,7 +287,11 @@ static void __init zone_sizes_init(unsigned long min, unsigned long max) #ifdef CONFIG_HAVE_ARCH_PFN_VALID int pfn_valid(unsigned long pfn) { - return memblock_is_map_memory(pfn << PAGE_SHIFT); + phys_addr_t addr = pfn << PAGE_SHIFT; + + if ((addr >> PAGE_SHIFT) != pfn) + return 0; + return memblock_is_map_memory(addr); } EXPORT_SYMBOL(pfn_valid); #endif diff --git a/arch/m68k/configs/mac_defconfig b/arch/m68k/configs/mac_defconfig index 930cc2965a11..c717bf879449 100644 --- a/arch/m68k/configs/mac_defconfig +++ b/arch/m68k/configs/mac_defconfig @@ -366,7 +366,7 @@ CONFIG_TCM_PSCSI=m CONFIG_ADB=y CONFIG_ADB_MACII=y CONFIG_ADB_IOP=y -CONFIG_ADB_PMU68K=y +CONFIG_ADB_PMU=y CONFIG_ADB_CUDA=y CONFIG_INPUT_ADBHID=y CONFIG_MAC_EMUMOUSEBTN=y diff --git a/arch/m68k/configs/multi_defconfig b/arch/m68k/configs/multi_defconfig index e7dd25300127..226c994ce794 100644 --- a/arch/m68k/configs/multi_defconfig +++ b/arch/m68k/configs/multi_defconfig @@ -400,7 +400,7 @@ CONFIG_TCM_PSCSI=m CONFIG_ADB=y CONFIG_ADB_MACII=y CONFIG_ADB_IOP=y -CONFIG_ADB_PMU68K=y +CONFIG_ADB_PMU=y CONFIG_ADB_CUDA=y CONFIG_INPUT_ADBHID=y CONFIG_MAC_EMUMOUSEBTN=y diff --git a/arch/m68k/mac/config.c b/arch/m68k/mac/config.c index b02d7254b73a..cd9317d53276 100644 --- a/arch/m68k/mac/config.c +++ b/arch/m68k/mac/config.c @@ -889,7 +889,7 @@ static void __init mac_identify(void) #ifdef CONFIG_ADB_CUDA find_via_cuda(); #endif -#ifdef CONFIG_ADB_PMU68K +#ifdef CONFIG_ADB_PMU find_via_pmu(); #endif } diff --git a/arch/m68k/mac/misc.c b/arch/m68k/mac/misc.c index 19e9d8eef1f2..3534aa6a4dc2 100644 --- a/arch/m68k/mac/misc.c +++ b/arch/m68k/mac/misc.c @@ -90,7 +90,7 @@ static void cuda_write_pram(int offset, __u8 data) } #endif /* CONFIG_ADB_CUDA */ -#ifdef CONFIG_ADB_PMU68K +#ifdef CONFIG_ADB_PMU static time64_t pmu_read_time(void) { struct adb_request req; @@ -142,7 +142,7 @@ static void pmu_write_pram(int offset, __u8 data) while (!req.complete) pmu_poll(); } -#endif /* CONFIG_ADB_PMU68K */ +#endif /* CONFIG_ADB_PMU */ /* * VIA PRAM/RTC access routines @@ -373,38 +373,6 @@ static void cuda_shutdown(void) } #endif /* CONFIG_ADB_CUDA */ -#ifdef CONFIG_ADB_PMU68K - -void pmu_restart(void) -{ - struct adb_request req; - if (pmu_request(&req, NULL, - 2, PMU_SET_INTR_MASK, PMU_INT_ADB|PMU_INT_TICK) < 0) - return; - while (!req.complete) - pmu_poll(); - if (pmu_request(&req, NULL, 1, PMU_RESET) < 0) - return; - while (!req.complete) - pmu_poll(); -} - -void pmu_shutdown(void) -{ - struct adb_request req; - if (pmu_request(&req, NULL, - 2, PMU_SET_INTR_MASK, PMU_INT_ADB|PMU_INT_TICK) < 0) - return; - while (!req.complete) - pmu_poll(); - if (pmu_request(&req, NULL, 5, PMU_SHUTDOWN, 'M', 'A', 'T', 'T') < 0) - return; - while (!req.complete) - pmu_poll(); -} - -#endif - /* *------------------------------------------------------------------- * Below this point are the generic routines; they'll dispatch to the @@ -429,7 +397,7 @@ void mac_pram_read(int offset, __u8 *buffer, int len) func = cuda_read_pram; break; #endif -#ifdef CONFIG_ADB_PMU68K +#ifdef CONFIG_ADB_PMU case MAC_ADB_PB2: func = pmu_read_pram; break; @@ -459,7 +427,7 @@ void mac_pram_write(int offset, __u8 *buffer, int len) func = cuda_write_pram; break; #endif -#ifdef CONFIG_ADB_PMU68K +#ifdef CONFIG_ADB_PMU case MAC_ADB_PB2: func = pmu_write_pram; break; @@ -483,9 +451,8 @@ void mac_poweroff(void) macintosh_config->adb_type == MAC_ADB_CUDA) { cuda_shutdown(); #endif -#ifdef CONFIG_ADB_PMU68K - } else if (macintosh_config->adb_type == MAC_ADB_PB1 - || macintosh_config->adb_type == MAC_ADB_PB2) { +#ifdef CONFIG_ADB_PMU + } else if (macintosh_config->adb_type == MAC_ADB_PB2) { pmu_shutdown(); #endif } @@ -525,9 +492,8 @@ void mac_reset(void) macintosh_config->adb_type == MAC_ADB_CUDA) { cuda_restart(); #endif -#ifdef CONFIG_ADB_PMU68K - } else if (macintosh_config->adb_type == MAC_ADB_PB1 - || macintosh_config->adb_type == MAC_ADB_PB2) { +#ifdef CONFIG_ADB_PMU + } else if (macintosh_config->adb_type == MAC_ADB_PB2) { pmu_restart(); #endif } else if (CPU_IS_030) { @@ -680,7 +646,7 @@ int mac_hwclk(int op, struct rtc_time *t) now = cuda_read_time(); break; #endif -#ifdef CONFIG_ADB_PMU68K +#ifdef CONFIG_ADB_PMU case MAC_ADB_PB2: now = pmu_read_time(); break; @@ -716,7 +682,7 @@ int mac_hwclk(int op, struct rtc_time *t) cuda_write_time(now); break; #endif -#ifdef CONFIG_ADB_PMU68K +#ifdef CONFIG_ADB_PMU case MAC_ADB_PB2: pmu_write_time(now); break; diff --git a/arch/powerpc/Kconfig b/arch/powerpc/Kconfig index 1c10ff0406f2..a80669209155 100644 --- a/arch/powerpc/Kconfig +++ b/arch/powerpc/Kconfig @@ -165,7 +165,7 @@ config PPC select GENERIC_CLOCKEVENTS_BROADCAST if SMP select GENERIC_CMOS_UPDATE select GENERIC_CPU_AUTOPROBE - select GENERIC_CPU_VULNERABILITIES if PPC_BOOK3S_64 + select GENERIC_CPU_VULNERABILITIES if PPC_BARRIER_NOSPEC select GENERIC_IRQ_SHOW select GENERIC_IRQ_SHOW_LEVEL select GENERIC_SMP_IDLE_THREAD @@ -197,6 +197,7 @@ config PPC select HAVE_IOREMAP_PROT select HAVE_IRQ_EXIT_ON_IRQ_STACK select HAVE_KERNEL_GZIP + select HAVE_KERNEL_XZ if PPC_BOOK3S select HAVE_KPROBES select HAVE_KPROBES_ON_FTRACE select HAVE_KRETPROBES @@ -225,6 +226,7 @@ config PPC select IRQ_DOMAIN select IRQ_FORCED_THREADING select MODULES_USE_ELF_RELA + select NEED_DMA_MAP_STATE if PPC64 || NOT_COHERENT_CACHE select NEED_SG_DMA_LENGTH select NO_BOOTMEM select OF @@ -240,6 +242,11 @@ config PPC # Please keep this list sorted alphabetically. # +config PPC_BARRIER_NOSPEC + bool + default y + depends on PPC_BOOK3S_64 || PPC_FSL_BOOK3E + config GENERIC_CSUM def_bool n @@ -876,9 +883,6 @@ config ZONE_DMA bool default y -config NEED_DMA_MAP_STATE - def_bool (PPC64 || NOT_COHERENT_CACHE) - config GENERIC_ISA_DMA bool depends on ISA_DMA_API diff --git a/arch/powerpc/Makefile b/arch/powerpc/Makefile index fb96206de317..8397c7bd5880 100644 --- a/arch/powerpc/Makefile +++ b/arch/powerpc/Makefile @@ -36,7 +36,7 @@ else KBUILD_DEFCONFIG := ppc64_defconfig endif -ifeq ($(CONFIG_PPC64),y) +ifdef CONFIG_PPC64 new_nm := $(shell if $(NM) --help 2>&1 | grep -- '--synthetic' > /dev/null; then echo y; else echo n; fi) ifeq ($(new_nm),y) @@ -74,7 +74,7 @@ KBUILD_LDFLAGS_MODULE += arch/powerpc/lib/crtsavres.o endif endif -ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) +ifdef CONFIG_CPU_LITTLE_ENDIAN KBUILD_CFLAGS += -mlittle-endian LDFLAGS += -EL LDEMULATION := lppc @@ -117,7 +117,7 @@ LDFLAGS_vmlinux-$(CONFIG_RELOCATABLE) := -pie LDFLAGS_vmlinux := $(LDFLAGS_vmlinux-y) LDFLAGS_vmlinux += $(call ld-option,--orphan-handling=warn) -ifeq ($(CONFIG_PPC64),y) +ifdef CONFIG_PPC64 ifeq ($(call cc-option-yn,-mcmodel=medium),y) # -mcmodel=medium breaks modules because it uses 32bit offsets from # the TOC pointer to create pointers where possible. Pointers into the @@ -134,7 +134,7 @@ endif endif CFLAGS-$(CONFIG_PPC64) := $(call cc-option,-mtraceback=no) -ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) +ifdef CONFIG_CPU_LITTLE_ENDIAN CFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mabi=elfv2,$(call cc-option,-mcall-aixdesc)) AFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mabi=elfv2) else @@ -148,8 +148,8 @@ CFLAGS-$(CONFIG_PPC64) += $(call cc-option,-mno-pointers-to-nested-functions) CFLAGS-$(CONFIG_PPC32) := -ffixed-r2 $(MULTIPLEWORD) CFLAGS-$(CONFIG_PPC32) += $(call cc-option,-mno-readonly-in-sdata) -ifeq ($(CONFIG_PPC_BOOK3S_64),y) -ifeq ($(CONFIG_CPU_LITTLE_ENDIAN),y) +ifdef CONFIG_PPC_BOOK3S_64 +ifdef CONFIG_CPU_LITTLE_ENDIAN CFLAGS-$(CONFIG_GENERIC_CPU) += -mcpu=power8 CFLAGS-$(CONFIG_GENERIC_CPU) += $(call cc-option,-mtune=power9,-mtune=power8) else @@ -164,16 +164,10 @@ ifdef CONFIG_MPROFILE_KERNEL CC_FLAGS_FTRACE := -pg -mprofile-kernel endif -CFLAGS-$(CONFIG_CELL_CPU) += $(call cc-option,-mcpu=cell) -CFLAGS-$(CONFIG_POWER5_CPU) += $(call cc-option,-mcpu=power5) -CFLAGS-$(CONFIG_POWER6_CPU) += $(call cc-option,-mcpu=power6) -CFLAGS-$(CONFIG_POWER7_CPU) += $(call cc-option,-mcpu=power7) -CFLAGS-$(CONFIG_POWER8_CPU) += $(call cc-option,-mcpu=power8) -CFLAGS-$(CONFIG_POWER9_CPU) += $(call cc-option,-mcpu=power9) -CFLAGS-$(CONFIG_PPC_8xx) += $(call cc-option,-mcpu=860) +CFLAGS-$(CONFIG_TARGET_CPU_BOOL) += $(call cc-option,-mcpu=$(CONFIG_TARGET_CPU)) # Altivec option not allowed with e500mc64 in GCC. -ifeq ($(CONFIG_ALTIVEC),y) +ifdef CONFIG_ALTIVEC E5500_CPU := -mcpu=powerpc64 else E5500_CPU := $(call cc-option,-mcpu=e500mc64,-mcpu=powerpc64) @@ -181,8 +175,8 @@ endif CFLAGS-$(CONFIG_E5500_CPU) += $(E5500_CPU) CFLAGS-$(CONFIG_E6500_CPU) += $(call cc-option,-mcpu=e6500,$(E5500_CPU)) -ifeq ($(CONFIG_PPC32),y) -ifeq ($(CONFIG_PPC_E500MC),y) +ifdef CONFIG_PPC32 +ifdef CONFIG_PPC_E500MC CFLAGS-y += $(call cc-option,-mcpu=e500mc,-mcpu=powerpc) else CFLAGS-$(CONFIG_E500) += $(call cc-option,-mcpu=8540 -msoft-float,-mcpu=powerpc) @@ -204,7 +198,7 @@ else CHECKFLAGS += -D__LITTLE_ENDIAN__ endif -ifeq ($(CONFIG_476FPE_ERR46),y) +ifdef CONFIG_476FPE_ERR46 KBUILD_LDFLAGS_MODULE += --ppc476-workaround \ -T $(srctree)/arch/powerpc/platforms/44x/ppc476_modules.lds endif @@ -231,12 +225,12 @@ KBUILD_CFLAGS += $(call cc-option,-fno-dwarf2-cfi-asm) # often slow when they are implemented at all KBUILD_CFLAGS += $(call cc-option,-mno-string) -ifeq ($(CONFIG_6xx),y) +ifdef CONFIG_6xx KBUILD_CFLAGS += -mcpu=powerpc endif # Work around a gcc code-gen bug with -fno-omit-frame-pointer. -ifeq ($(CONFIG_FUNCTION_TRACER),y) +ifdef CONFIG_FUNCTION_TRACER KBUILD_CFLAGS += -mno-sched-epilog endif @@ -355,6 +349,21 @@ mpc86xx_smp_defconfig: $(call merge_into_defconfig,mpc86xx_basic_defconfig,\ 86xx-smp 86xx-hw fsl-emb-nonhw) +PHONY += ppc32_allmodconfig +ppc32_allmodconfig: + $(Q)$(MAKE) KCONFIG_ALLCONFIG=$(srctree)/arch/powerpc/configs/book3s_32.config \ + -f $(srctree)/Makefile allmodconfig + +PHONY += ppc64le_allmodconfig +ppc64le_allmodconfig: + $(Q)$(MAKE) KCONFIG_ALLCONFIG=$(srctree)/arch/powerpc/configs/le.config \ + -f $(srctree)/Makefile allmodconfig + +PHONY += ppc64_book3e_allmodconfig +ppc64_book3e_allmodconfig: + $(Q)$(MAKE) KCONFIG_ALLCONFIG=$(srctree)/arch/powerpc/configs/85xx-64bit.config \ + -f $(srctree)/Makefile allmodconfig + define archhelp @echo '* zImage - Build default images selected by kernel config' @echo ' zImage.* - Compressed kernel image (arch/$(ARCH)/boot/zImage.*)' @@ -381,7 +390,7 @@ install: $(Q)$(MAKE) $(build)=$(boot) install vdso_install: -ifeq ($(CONFIG_PPC64),y) +ifdef CONFIG_PPC64 $(Q)$(MAKE) $(build)=arch/$(ARCH)/kernel/vdso64 $@ endif $(Q)$(MAKE) $(build)=arch/$(ARCH)/kernel/vdso32 $@ diff --git a/arch/powerpc/boot/Makefile b/arch/powerpc/boot/Makefile index deea20c334df..0fb96c26136f 100644 --- a/arch/powerpc/boot/Makefile +++ b/arch/powerpc/boot/Makefile @@ -354,7 +354,7 @@ image-$(CONFIG_AMIGAONE) += cuImage.amigaone # For 32-bit powermacs, build the COFF and miboot images # as well as the ELF images. -ifeq ($(CONFIG_PPC32),y) +ifdef CONFIG_PPC32 image-$(CONFIG_PPC_PMAC) += zImage.coff zImage.miboot endif diff --git a/arch/powerpc/boot/dts/ac14xx.dts b/arch/powerpc/boot/dts/ac14xx.dts index 83bcfd865167..0be5c4f3265d 100644 --- a/arch/powerpc/boot/dts/ac14xx.dts +++ b/arch/powerpc/boot/dts/ac14xx.dts @@ -176,12 +176,12 @@ clock-frequency = <400000>; at24@30 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x30>; }; at24@31 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x31>; }; @@ -191,42 +191,42 @@ }; at24@50 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x50>; }; at24@51 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x51>; }; at24@52 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x52>; }; at24@53 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x53>; }; at24@54 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x54>; }; at24@55 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x55>; }; at24@56 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x56>; }; at24@57 { - compatible = "at24,24c01"; + compatible = "atmel,24c01"; reg = <0x57>; }; diff --git a/arch/powerpc/boot/dts/fsl/kmcent2.dts b/arch/powerpc/boot/dts/fsl/kmcent2.dts index 5922c1ea0e96..3094df05f5ea 100644 --- a/arch/powerpc/boot/dts/fsl/kmcent2.dts +++ b/arch/powerpc/boot/dts/fsl/kmcent2.dts @@ -130,7 +130,7 @@ #size-cells = <0>; eeprom@54 { - compatible = "24c02"; + compatible = "atmel,24c02"; reg = <0x54>; pagesize = <2>; read-only; diff --git a/arch/powerpc/boot/dts/fsl/t2080rdb.dts b/arch/powerpc/boot/dts/fsl/t2080rdb.dts index 836e4c965b22..55c0210a771d 100644 --- a/arch/powerpc/boot/dts/fsl/t2080rdb.dts +++ b/arch/powerpc/boot/dts/fsl/t2080rdb.dts @@ -97,12 +97,12 @@ mdio@fd000 { xg_cs4315_phy1: ethernet-phy@c { - compatible = "ethernet-phy-ieee802.3-c45"; + compatible = "ethernet-phy-id13e5.1002"; reg = <0xc>; }; xg_cs4315_phy2: ethernet-phy@d { - compatible = "ethernet-phy-ieee802.3-c45"; + compatible = "ethernet-phy-id13e5.1002"; reg = <0xd>; }; diff --git a/arch/powerpc/boot/dts/fsl/t4240rdb.dts b/arch/powerpc/boot/dts/fsl/t4240rdb.dts index 15eb0a3f7290..a56a705d41f7 100644 --- a/arch/powerpc/boot/dts/fsl/t4240rdb.dts +++ b/arch/powerpc/boot/dts/fsl/t4240rdb.dts @@ -267,22 +267,22 @@ mdio@fd000 { xfiphy1: ethernet-phy@10 { - compatible = "ethernet-phy-ieee802.3-c45"; + compatible = "ethernet-phy-id13e5.1002"; reg = <0x10>; }; xfiphy2: ethernet-phy@11 { - compatible = "ethernet-phy-ieee802.3-c45"; + compatible = "ethernet-phy-id13e5.1002"; reg = <0x11>; }; xfiphy3: ethernet-phy@13 { - compatible = "ethernet-phy-ieee802.3-c45"; + compatible = "ethernet-phy-id13e5.1002"; reg = <0x13>; }; xfiphy4: ethernet-phy@12 { - compatible = "ethernet-phy-ieee802.3-c45"; + compatible = "ethernet-phy-id13e5.1002"; reg = <0x12>; }; }; diff --git a/arch/powerpc/boot/dts/pdm360ng.dts b/arch/powerpc/boot/dts/pdm360ng.dts index 445b88114009..df1283b63d9b 100644 --- a/arch/powerpc/boot/dts/pdm360ng.dts +++ b/arch/powerpc/boot/dts/pdm360ng.dts @@ -98,7 +98,7 @@ fsl,preserve-clocking; eeprom@50 { - compatible = "at,24c01"; + compatible = "atmel,24c01"; reg = <0x50>; }; diff --git a/arch/powerpc/configs/book3s_32.config b/arch/powerpc/configs/book3s_32.config new file mode 100644 index 000000000000..8721eb7b1294 --- /dev/null +++ b/arch/powerpc/configs/book3s_32.config @@ -0,0 +1,2 @@ +CONFIG_PPC64=n +CONFIG_PPC_BOOK3S_32=y diff --git a/arch/powerpc/configs/dpaa.config b/arch/powerpc/configs/dpaa.config index 2fe76f5e938a..4ffacafe4036 100644 --- a/arch/powerpc/configs/dpaa.config +++ b/arch/powerpc/configs/dpaa.config @@ -2,3 +2,4 @@ CONFIG_FSL_DPAA=y CONFIG_FSL_PAMU=y CONFIG_FSL_FMAN=y CONFIG_FSL_DPAA_ETH=y +CONFIG_CORTINA_PHY=y diff --git a/arch/powerpc/crypto/md5-asm.S b/arch/powerpc/crypto/md5-asm.S index 10cdf5bceebb..1834065362c7 100644 --- a/arch/powerpc/crypto/md5-asm.S +++ b/arch/powerpc/crypto/md5-asm.S @@ -11,6 +11,7 @@ */ #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/asm-compat.h> #define rHP r3 #define rWP r4 diff --git a/arch/powerpc/crypto/sha1-powerpc-asm.S b/arch/powerpc/crypto/sha1-powerpc-asm.S index c8951ce0dcc4..23e248beff71 100644 --- a/arch/powerpc/crypto/sha1-powerpc-asm.S +++ b/arch/powerpc/crypto/sha1-powerpc-asm.S @@ -7,6 +7,7 @@ #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/asm-compat.h> #ifdef __BIG_ENDIAN__ #define LWZ(rt, d, ra) \ diff --git a/arch/powerpc/include/asm/asm-405.h b/arch/powerpc/include/asm/asm-405.h new file mode 100644 index 000000000000..7270d3ae7c8e --- /dev/null +++ b/arch/powerpc/include/asm/asm-405.h @@ -0,0 +1,19 @@ +#ifndef _ASM_POWERPC_ASM_405_H +#define _ASM_POWERPC_ASM_405_H + +#include <asm/asm-const.h> + +#ifdef __KERNEL__ +#ifdef CONFIG_IBM405_ERR77 +/* Erratum #77 on the 405 means we need a sync or dcbt before every + * stwcx. The old ATOMIC_SYNC_FIX covered some but not all of this. + */ +#define PPC405_ERR77(ra,rb) stringify_in_c(dcbt ra, rb;) +#define PPC405_ERR77_SYNC stringify_in_c(sync;) +#else +#define PPC405_ERR77(ra,rb) +#define PPC405_ERR77_SYNC +#endif +#endif + +#endif /* _ASM_POWERPC_ASM_405_H */ diff --git a/arch/powerpc/include/asm/asm-compat.h b/arch/powerpc/include/asm/asm-compat.h index 7f2a7702596c..19b70c5b5f18 100644 --- a/arch/powerpc/include/asm/asm-compat.h +++ b/arch/powerpc/include/asm/asm-compat.h @@ -1,21 +1,10 @@ #ifndef _ASM_POWERPC_ASM_COMPAT_H #define _ASM_POWERPC_ASM_COMPAT_H +#include <asm/asm-const.h> #include <asm/types.h> #include <asm/ppc-opcode.h> -#ifdef __ASSEMBLY__ -# define stringify_in_c(...) __VA_ARGS__ -# define ASM_CONST(x) x -#else -/* This version of stringify will deal with commas... */ -# define __stringify_in_c(...) #__VA_ARGS__ -# define stringify_in_c(...) __stringify_in_c(__VA_ARGS__) " " -# define __ASM_CONST(x) x##UL -# define ASM_CONST(x) __ASM_CONST(x) -#endif - - #ifdef __powerpc64__ /* operations for longs and pointers */ @@ -70,17 +59,4 @@ #endif -#ifdef __KERNEL__ -#ifdef CONFIG_IBM405_ERR77 -/* Erratum #77 on the 405 means we need a sync or dcbt before every - * stwcx. The old ATOMIC_SYNC_FIX covered some but not all of this. - */ -#define PPC405_ERR77(ra,rb) stringify_in_c(dcbt ra, rb;) -#define PPC405_ERR77_SYNC stringify_in_c(sync;) -#else -#define PPC405_ERR77(ra,rb) -#define PPC405_ERR77_SYNC -#endif -#endif - #endif /* _ASM_POWERPC_ASM_COMPAT_H */ diff --git a/arch/powerpc/include/asm/asm-const.h b/arch/powerpc/include/asm/asm-const.h new file mode 100644 index 000000000000..082c1538c562 --- /dev/null +++ b/arch/powerpc/include/asm/asm-const.h @@ -0,0 +1,14 @@ +#ifndef _ASM_POWERPC_ASM_CONST_H +#define _ASM_POWERPC_ASM_CONST_H + +#ifdef __ASSEMBLY__ +# define stringify_in_c(...) __VA_ARGS__ +# define ASM_CONST(x) x +#else +/* This version of stringify will deal with commas... */ +# define __stringify_in_c(...) #__VA_ARGS__ +# define stringify_in_c(...) __stringify_in_c(__VA_ARGS__) " " +# define __ASM_CONST(x) x##UL +# define ASM_CONST(x) __ASM_CONST(x) +#endif +#endif /* _ASM_POWERPC_ASM_CONST_H */ diff --git a/arch/powerpc/include/asm/asm-prototypes.h b/arch/powerpc/include/asm/asm-prototypes.h index 7841b8a60657..1f4691ce4126 100644 --- a/arch/powerpc/include/asm/asm-prototypes.h +++ b/arch/powerpc/include/asm/asm-prototypes.h @@ -48,8 +48,8 @@ void __trace_opal_exit(long opcode, unsigned long retval); /* VMX copying */ int enter_vmx_usercopy(void); int exit_vmx_usercopy(void); -int enter_vmx_copy(void); -void * exit_vmx_copy(void *dest); +int enter_vmx_ops(void); +void *exit_vmx_ops(void *dest); /* Traps */ long machine_check_early(struct pt_regs *regs); @@ -143,4 +143,11 @@ struct kvm_vcpu; void _kvmppc_restore_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr); void _kvmppc_save_tm_pr(struct kvm_vcpu *vcpu, u64 guest_msr); +/* Patch sites */ +extern s32 patch__call_flush_count_cache; +extern s32 patch__flush_count_cache_return; +extern s32 patch__memset_nocache, patch__memcpy_nocache; + +extern long flush_count_cache; + #endif /* _ASM_POWERPC_ASM_PROTOTYPES_H */ diff --git a/arch/powerpc/include/asm/atomic.h b/arch/powerpc/include/asm/atomic.h index 963abf8bf1c0..52eafaf74054 100644 --- a/arch/powerpc/include/asm/atomic.h +++ b/arch/powerpc/include/asm/atomic.h @@ -10,6 +10,7 @@ #include <linux/types.h> #include <asm/cmpxchg.h> #include <asm/barrier.h> +#include <asm/asm-405.h> #define ATOMIC_INIT(i) { (i) } diff --git a/arch/powerpc/include/asm/barrier.h b/arch/powerpc/include/asm/barrier.h index f67b3f6e36be..fbe8df433019 100644 --- a/arch/powerpc/include/asm/barrier.h +++ b/arch/powerpc/include/asm/barrier.h @@ -5,6 +5,8 @@ #ifndef _ASM_POWERPC_BARRIER_H #define _ASM_POWERPC_BARRIER_H +#include <asm/asm-const.h> + /* * Memory barrier. * The sync instruction guarantees that all memory accesses initiated @@ -77,19 +79,25 @@ do { \ }) #ifdef CONFIG_PPC_BOOK3S_64 +#define NOSPEC_BARRIER_SLOT nop +#elif defined(CONFIG_PPC_FSL_BOOK3E) +#define NOSPEC_BARRIER_SLOT nop; nop +#endif + +#ifdef CONFIG_PPC_BARRIER_NOSPEC /* * Prevent execution of subsequent instructions until preceding branches have * been fully resolved and are no longer executing speculatively. */ -#define barrier_nospec_asm NOSPEC_BARRIER_FIXUP_SECTION; nop +#define barrier_nospec_asm NOSPEC_BARRIER_FIXUP_SECTION; NOSPEC_BARRIER_SLOT // This also acts as a compiler barrier due to the memory clobber. #define barrier_nospec() asm (stringify_in_c(barrier_nospec_asm) ::: "memory") -#else /* !CONFIG_PPC_BOOK3S_64 */ +#else /* !CONFIG_PPC_BARRIER_NOSPEC */ #define barrier_nospec_asm #define barrier_nospec() -#endif +#endif /* CONFIG_PPC_BARRIER_NOSPEC */ #include <asm-generic/barrier.h> diff --git a/arch/powerpc/include/asm/bitops.h b/arch/powerpc/include/asm/bitops.h index b750ffef83c7..ff71566dadee 100644 --- a/arch/powerpc/include/asm/bitops.h +++ b/arch/powerpc/include/asm/bitops.h @@ -45,6 +45,7 @@ #include <linux/compiler.h> #include <asm/asm-compat.h> #include <asm/synch.h> +#include <asm/asm-405.h> /* PPC bit number conversion */ #define PPC_BITLSHIFT(be) (BITS_PER_LONG - 1 - (be)) diff --git a/arch/powerpc/include/asm/book3s/32/pgtable.h b/arch/powerpc/include/asm/book3s/32/pgtable.h index 02f5acd7ccc4..751cf931bb3f 100644 --- a/arch/powerpc/include/asm/book3s/32/pgtable.h +++ b/arch/powerpc/include/asm/book3s/32/pgtable.h @@ -84,17 +84,12 @@ * of RAM. -- Cort */ #define VMALLOC_OFFSET (0x1000000) /* 16M */ -#ifdef PPC_PIN_SIZE -#define VMALLOC_START (((_ALIGN((long)high_memory, PPC_PIN_SIZE) + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1))) -#else #define VMALLOC_START ((((long)high_memory + VMALLOC_OFFSET) & ~(VMALLOC_OFFSET-1))) -#endif #define VMALLOC_END ioremap_bot #ifndef __ASSEMBLY__ #include <linux/sched.h> #include <linux/threads.h> -#include <asm/io.h> /* For sub-arch specific PPC_PIN_SIZE */ extern unsigned long ioremap_bot; @@ -164,7 +159,6 @@ static inline unsigned long pte_update(pte_t *p, 1: lwarx %0,0,%3\n\ andc %1,%0,%4\n\ or %1,%1,%5\n" - PPC405_ERR77(0,%3) " stwcx. %1,0,%3\n\ bne- 1b" : "=&r" (old), "=&r" (tmp), "=m" (*p) @@ -186,7 +180,6 @@ static inline unsigned long long pte_update(pte_t *p, lwzx %0,0,%3\n\ andc %1,%L0,%5\n\ or %1,%1,%6\n" - PPC405_ERR77(0,%3) " stwcx. %1,0,%4\n\ bne- 1b" : "=&r" (old), "=&r" (tmp), "=m" (*p) diff --git a/arch/powerpc/include/asm/book3s/32/tlbflush.h b/arch/powerpc/include/asm/book3s/32/tlbflush.h new file mode 100644 index 000000000000..068085b709fb --- /dev/null +++ b/arch/powerpc/include/asm/book3s/32/tlbflush.h @@ -0,0 +1,25 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_32_TLBFLUSH_H +#define _ASM_POWERPC_BOOK3S_32_TLBFLUSH_H + +#define MMU_NO_CONTEXT (0) +/* + * TLB flushing for "classic" hash-MMU 32-bit CPUs, 6xx, 7xx, 7xxx + */ +extern void flush_tlb_mm(struct mm_struct *mm); +extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); +extern void flush_tlb_page_nohash(struct vm_area_struct *vma, unsigned long addr); +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); +static inline void local_flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ + flush_tlb_page(vma, vmaddr); +} +static inline void local_flush_tlb_mm(struct mm_struct *mm) +{ + flush_tlb_mm(mm); +} + +#endif /* _ASM_POWERPC_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index c81793d47af9..f82ee8a3b561 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -137,10 +137,9 @@ extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index); shift = mmu_psize_defs[psize].shift; \ for (index = 0; vpn < __end; index++, \ vpn += (1L << (shift - VPN_SHIFT))) { \ - if (!__split || __rpte_sub_valid(rpte, index)) \ - do { + if (!__split || __rpte_sub_valid(rpte, index)) -#define pte_iterate_hashed_end() } while(0); } } while(0) +#define pte_iterate_hashed_end() } } while(0) #define pte_pagesize_index(mm, addr, pte) \ (((pte) & H_PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K) diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index 0387b155f13d..d52a51b2ce7b 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -3,6 +3,8 @@ #define _ASM_POWERPC_BOOK3S_64_HASH_H #ifdef __KERNEL__ +#include <asm/asm-const.h> + /* * Common bits between 4K and 64K pages in a linux-style PTE. * Additional bits may be defined in pgtable-hash64-*.h diff --git a/arch/powerpc/include/asm/book3s/64/hugetlb.h b/arch/powerpc/include/asm/book3s/64/hugetlb.h index c459f937d484..50888388a359 100644 --- a/arch/powerpc/include/asm/book3s/64/hugetlb.h +++ b/arch/powerpc/include/asm/book3s/64/hugetlb.h @@ -32,26 +32,6 @@ static inline int hstate_get_psize(struct hstate *hstate) } } -#define arch_make_huge_pte arch_make_huge_pte -static inline pte_t arch_make_huge_pte(pte_t entry, struct vm_area_struct *vma, - struct page *page, int writable) -{ - unsigned long page_shift; - - if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) - return entry; - - page_shift = huge_page_shift(hstate_vma(vma)); - /* - * We don't support 1G hugetlb pages yet. - */ - VM_WARN_ON(page_shift == mmu_psize_defs[MMU_PAGE_1G].shift); - if (page_shift == mmu_psize_defs[MMU_PAGE_2M].shift) - return __pte(pte_val(entry) | R_PAGE_LARGE); - else - return entry; -} - #ifdef CONFIG_ARCH_HAS_GIGANTIC_PAGE static inline bool gigantic_page_supported(void) { diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h index 50ed64fba4ae..b3520b549cba 100644 --- a/arch/powerpc/include/asm/book3s/64/mmu-hash.h +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -12,9 +12,9 @@ * 2 of the License, or (at your option) any later version. */ -#include <asm/asm-compat.h> #include <asm/page.h> #include <asm/bug.h> +#include <asm/asm-const.h> /* * This is necessary to get the definition of PGTABLE_RANGE which we @@ -364,6 +364,16 @@ static inline unsigned long hpte_new_to_old_r(unsigned long r) return r & ~HPTE_R_3_0_SSIZE_MASK; } +static inline unsigned long hpte_get_old_v(struct hash_pte *hptep) +{ + unsigned long hpte_v; + + hpte_v = be64_to_cpu(hptep->v); + if (cpu_has_feature(CPU_FTR_ARCH_300)) + hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); + return hpte_v; +} + /* * This function sets the AVPN and L fields of the HPTE appropriately * using the base page size and actual page size. @@ -487,6 +497,9 @@ extern void hpte_init_native(void); extern void slb_initialize(void); extern void slb_flush_and_rebolt(void); +void slb_flush_all_realmode(void); +void __slb_restore_bolted_realmode(void); +void slb_restore_bolted_realmode(void); extern void slb_vmalloc_update(void); extern void slb_set_size(u16 size); diff --git a/arch/powerpc/include/asm/book3s/64/pgalloc.h b/arch/powerpc/include/asm/book3s/64/pgalloc.h index 01ee40f11f3a..391ed2c3b697 100644 --- a/arch/powerpc/include/asm/book3s/64/pgalloc.h +++ b/arch/powerpc/include/asm/book3s/64/pgalloc.h @@ -9,6 +9,7 @@ #include <linux/slab.h> #include <linux/cpumask.h> +#include <linux/kmemleak.h> #include <linux/percpu.h> struct vmemmap_backing { @@ -83,6 +84,13 @@ static inline pgd_t *pgd_alloc(struct mm_struct *mm) pgd = kmem_cache_alloc(PGT_CACHE(PGD_INDEX_SIZE), pgtable_gfp_flags(mm, GFP_KERNEL)); /* + * Don't scan the PGD for pointers, it contains references to PUDs but + * those references are not full pointers and so can't be recognised by + * kmemleak. + */ + kmemleak_no_scan(pgd); + + /* * With hugetlb, we don't clear the second half of the page table. * If we share the same slab cache with the pmd or pud level table, * we need to make sure we zero out the full table on alloc. @@ -110,8 +118,19 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { - return kmem_cache_alloc(PGT_CACHE(PUD_CACHE_INDEX), - pgtable_gfp_flags(mm, GFP_KERNEL)); + pud_t *pud; + + pud = kmem_cache_alloc(PGT_CACHE(PUD_CACHE_INDEX), + pgtable_gfp_flags(mm, GFP_KERNEL)); + /* + * Tell kmemleak to ignore the PUD, that means don't scan it for + * pointers and don't consider it a leak. PUDs are typically only + * referred to by their PGD, but kmemleak is not able to recognise those + * as pointers, leading to false leak reports. + */ + kmemleak_ignore(pud); + + return pud; } static inline void pud_free(struct mm_struct *mm, pud_t *pud) @@ -208,4 +227,11 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t table, #define check_pgt_cache() do { } while (0) +extern atomic_long_t direct_pages_count[MMU_PAGE_COUNT]; +static inline void update_page_count(int psize, long count) +{ + if (IS_ENABLED(CONFIG_PROC_FS)) + atomic_long_add(count, &direct_pages_count[psize]); +} + #endif /* _ASM_POWERPC_BOOK3S_64_PGALLOC_H */ diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 42aafba7a308..676118743a06 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -479,9 +479,8 @@ static inline pte_t ptep_get_and_clear_full(struct mm_struct *mm, { if (full && radix_enabled()) { /* - * Let's skip the DD1 style pte update here. We know that - * this is a full mm pte clear and hence can be sure there is - * no parallel set_pte. + * We know that this is a full mm pte clear and + * hence can be sure there is no parallel set_pte. */ return radix__ptep_get_and_clear_full(mm, addr, ptep, full); } diff --git a/arch/powerpc/include/asm/book3s/64/radix.h b/arch/powerpc/include/asm/book3s/64/radix.h index ef9f96742ce1..7d1a3d1543fc 100644 --- a/arch/powerpc/include/asm/book3s/64/radix.h +++ b/arch/powerpc/include/asm/book3s/64/radix.h @@ -2,6 +2,8 @@ #ifndef _ASM_POWERPC_PGTABLE_RADIX_H #define _ASM_POWERPC_PGTABLE_RADIX_H +#include <asm/asm-const.h> + #ifndef __ASSEMBLY__ #include <asm/cmpxchg.h> #endif @@ -12,12 +14,6 @@ #include <asm/book3s/64/radix-4k.h> #endif -/* - * For P9 DD1 only, we need to track whether the pte's huge. - */ -#define R_PAGE_LARGE _RPAGE_RSV1 - - #ifndef __ASSEMBLY__ #include <asm/book3s/64/tlbflush-radix.h> #include <asm/cpu_has_feature.h> @@ -36,6 +32,9 @@ #define RADIX_PUD_BAD_BITS 0x60000000000000e0UL #define RADIX_PGD_BAD_BITS 0x60000000000000e0UL +#define RADIX_PMD_SHIFT (PAGE_SHIFT + RADIX_PTE_INDEX_SIZE) +#define RADIX_PUD_SHIFT (RADIX_PMD_SHIFT + RADIX_PMD_INDEX_SIZE) +#define RADIX_PGD_SHIFT (RADIX_PUD_SHIFT + RADIX_PUD_INDEX_SIZE) /* * Size of EA range mapped by our pagetables. */ @@ -154,20 +153,7 @@ static inline unsigned long radix__pte_update(struct mm_struct *mm, { unsigned long old_pte; - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { - - unsigned long new_pte; - - old_pte = __radix_pte_update(ptep, ~0ul, 0); - /* - * new value of pte - */ - new_pte = (old_pte | set) & ~clr; - radix__flush_tlb_pte_p9_dd1(old_pte, mm, addr); - if (new_pte) - __radix_pte_update(ptep, 0, new_pte); - } else - old_pte = __radix_pte_update(ptep, clr, set); + old_pte = __radix_pte_update(ptep, clr, set); if (!huge) assert_pte_locked(mm, addr); @@ -253,8 +239,6 @@ static inline int radix__pmd_trans_huge(pmd_t pmd) static inline pmd_t radix__pmd_mkhuge(pmd_t pmd) { - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - return __pmd(pmd_val(pmd) | _PAGE_PTE | R_PAGE_LARGE); return __pmd(pmd_val(pmd) | _PAGE_PTE); } @@ -285,18 +269,14 @@ static inline unsigned long radix__get_tree_size(void) unsigned long rts_field; /* * We support 52 bits, hence: - * DD1 52-28 = 24, 0b11000 - * Others 52-31 = 21, 0b10101 + * bits 52 - 31 = 21, 0b10101 * RTS encoding details * bits 0 - 3 of rts -> bits 6 - 8 unsigned long * bits 4 - 5 of rts -> bits 62 - 63 of unsigned long */ - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - rts_field = (0x3UL << 61); - else { - rts_field = (0x5UL << 5); /* 6 - 8 bits */ - rts_field |= (0x2UL << 61); - } + rts_field = (0x5UL << 5); /* 6 - 8 bits */ + rts_field |= (0x2UL << 61); + return rts_field; } diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h index ef5c3f2994c9..1154a6dc6d26 100644 --- a/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-radix.h @@ -48,8 +48,6 @@ extern void radix__flush_tlb_page_psize(struct mm_struct *mm, unsigned long vmad extern void radix__flush_tlb_pwc(struct mmu_gather *tlb, unsigned long addr); extern void radix__flush_tlb_collapsed_pmd(struct mm_struct *mm, unsigned long addr); extern void radix__flush_tlb_all(void); -extern void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm, - unsigned long address); extern void radix__flush_tlb_lpid_page(unsigned int lpid, unsigned long addr, diff --git a/arch/powerpc/include/asm/book3s/tlbflush.h b/arch/powerpc/include/asm/book3s/tlbflush.h new file mode 100644 index 000000000000..dec11de41055 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/tlbflush.h @@ -0,0 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_BOOK3S_TLBFLUSH_H +#define _ASM_POWERPC_BOOK3S_TLBFLUSH_H + +#ifdef CONFIG_PPC64 +#include <asm/book3s/64/tlbflush.h> +#else +#include <asm/book3s/32/tlbflush.h> +#endif + +#endif /* _ASM_POWERPC_BOOK3S_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/cacheflush.h b/arch/powerpc/include/asm/cacheflush.h index 0d72ec75da63..d5a8d7bf0759 100644 --- a/arch/powerpc/include/asm/cacheflush.h +++ b/arch/powerpc/include/asm/cacheflush.h @@ -11,7 +11,6 @@ #include <linux/mm.h> #include <asm/cputable.h> -#include <asm/cpu_has_feature.h> /* * No cache flushing is required when address mappings are changed, diff --git a/arch/powerpc/include/asm/cmpxchg.h b/arch/powerpc/include/asm/cmpxchg.h index 9b001f1f6b32..27183871eb3b 100644 --- a/arch/powerpc/include/asm/cmpxchg.h +++ b/arch/powerpc/include/asm/cmpxchg.h @@ -5,8 +5,8 @@ #ifdef __KERNEL__ #include <linux/compiler.h> #include <asm/synch.h> -#include <asm/asm-compat.h> #include <linux/bug.h> +#include <asm/asm-405.h> #ifdef __BIG_ENDIAN #define BITOFF_CAL(size, off) ((sizeof(u32) - size - off) * BITS_PER_BYTE) diff --git a/arch/powerpc/include/asm/code-patching-asm.h b/arch/powerpc/include/asm/code-patching-asm.h new file mode 100644 index 000000000000..ed7b1448493a --- /dev/null +++ b/arch/powerpc/include/asm/code-patching-asm.h @@ -0,0 +1,18 @@ +/* SPDX-License-Identifier: GPL-2.0+ */ +/* + * Copyright 2018, Michael Ellerman, IBM Corporation. + */ +#ifndef _ASM_POWERPC_CODE_PATCHING_ASM_H +#define _ASM_POWERPC_CODE_PATCHING_ASM_H + +/* Define a "site" that can be patched */ +.macro patch_site label name + .pushsection ".rodata" + .balign 4 + .global \name +\name: + .4byte \label - . + .popsection +.endm + +#endif /* _ASM_POWERPC_CODE_PATCHING_ASM_H */ diff --git a/arch/powerpc/include/asm/code-patching.h b/arch/powerpc/include/asm/code-patching.h index 812535f40124..31733a95bbd0 100644 --- a/arch/powerpc/include/asm/code-patching.h +++ b/arch/powerpc/include/asm/code-patching.h @@ -14,6 +14,7 @@ #include <asm/ppc-opcode.h> #include <linux/string.h> #include <linux/kallsyms.h> +#include <asm/asm-compat.h> /* Flags for create_branch: * "b" == create_branch(addr, target, 0); @@ -32,6 +33,8 @@ unsigned int create_cond_branch(const unsigned int *addr, int patch_branch(unsigned int *addr, unsigned long target, int flags); int patch_instruction(unsigned int *addr, unsigned int instr); int raw_patch_instruction(unsigned int *addr, unsigned int instr); +int patch_instruction_site(s32 *addr, unsigned int instr); +int patch_branch_site(s32 *site, unsigned long target, int flags); int instr_is_relative_branch(unsigned int instr); int instr_is_relative_link_branch(unsigned int instr); diff --git a/arch/powerpc/include/asm/cpuidle.h b/arch/powerpc/include/asm/cpuidle.h index e210a83eb196..43e5f31fe64d 100644 --- a/arch/powerpc/include/asm/cpuidle.h +++ b/arch/powerpc/include/asm/cpuidle.h @@ -79,6 +79,19 @@ struct stop_sprs { u64 mmcra; }; +#define PNV_IDLE_NAME_LEN 16 +struct pnv_idle_states_t { + char name[PNV_IDLE_NAME_LEN]; + u32 latency_ns; + u32 residency_ns; + u64 psscr_val; + u64 psscr_mask; + u32 flags; + bool valid; +}; + +extern struct pnv_idle_states_t *pnv_idle_states; +extern int nr_pnv_idle_states; extern u32 pnv_fastsleep_workaround_at_entry[]; extern u32 pnv_fastsleep_workaround_at_exit[]; diff --git a/arch/powerpc/include/asm/cputable.h b/arch/powerpc/include/asm/cputable.h index 9c0a3083571b..29f49a35d6ee 100644 --- a/arch/powerpc/include/asm/cputable.h +++ b/arch/powerpc/include/asm/cputable.h @@ -4,9 +4,8 @@ #include <linux/types.h> -#include <asm/asm-compat.h> -#include <asm/feature-fixups.h> #include <uapi/asm/cputable.h> +#include <asm/asm-const.h> #ifndef __ASSEMBLY__ @@ -210,7 +209,6 @@ static inline void cpu_feature_keys_init(void) { } #define CPU_FTR_DAWR LONG_ASM_CONST(0x0000008000000000) #define CPU_FTR_DABRX LONG_ASM_CONST(0x0000010000000000) #define CPU_FTR_PMAO_BUG LONG_ASM_CONST(0x0000020000000000) -#define CPU_FTR_POWER9_DD1 LONG_ASM_CONST(0x0000040000000000) #define CPU_FTR_POWER9_DD2_1 LONG_ASM_CONST(0x0000080000000000) #define CPU_FTR_P9_TM_HV_ASSIST LONG_ASM_CONST(0x0000100000000000) #define CPU_FTR_P9_TM_XER_SO_BUG LONG_ASM_CONST(0x0000200000000000) @@ -452,7 +450,6 @@ static inline void cpu_feature_keys_init(void) { } CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_DAWR | \ CPU_FTR_ARCH_207S | CPU_FTR_TM_COMP | CPU_FTR_PKEY) #define CPU_FTRS_POWER8E (CPU_FTRS_POWER8 | CPU_FTR_PMAO_BUG) -#define CPU_FTRS_POWER8_DD1 (CPU_FTRS_POWER8 & ~CPU_FTR_DBELL) #define CPU_FTRS_POWER9 (CPU_FTR_LWSYNC | \ CPU_FTR_PPCAS_ARCH_V2 | CPU_FTR_CTRL | CPU_FTR_ARCH_206 |\ CPU_FTR_MMCRA | CPU_FTR_SMT | \ @@ -464,8 +461,6 @@ static inline void cpu_feature_keys_init(void) { } CPU_FTR_DBELL | CPU_FTR_HAS_PPR | CPU_FTR_ARCH_207S | \ CPU_FTR_TM_COMP | CPU_FTR_ARCH_300 | CPU_FTR_PKEY | \ CPU_FTR_P9_TLBIE_BUG | CPU_FTR_P9_TIDR) -#define CPU_FTRS_POWER9_DD1 ((CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD1) & \ - (~CPU_FTR_SAO)) #define CPU_FTRS_POWER9_DD2_0 CPU_FTRS_POWER9 #define CPU_FTRS_POWER9_DD2_1 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1) #define CPU_FTRS_POWER9_DD2_2 (CPU_FTRS_POWER9 | CPU_FTR_POWER9_DD2_1 | \ @@ -488,17 +483,15 @@ static inline void cpu_feature_keys_init(void) { } #ifdef CONFIG_CPU_LITTLE_ENDIAN #define CPU_FTRS_POSSIBLE \ (CPU_FTRS_POWER7 | CPU_FTRS_POWER8E | CPU_FTRS_POWER8 | \ - CPU_FTRS_POWER8_DD1 | CPU_FTR_ALTIVEC_COMP | CPU_FTR_VSX_COMP | \ - CPU_FTRS_POWER9 | CPU_FTRS_POWER9_DD1 | CPU_FTRS_POWER9_DD2_1 | \ - CPU_FTRS_POWER9_DD2_2) + CPU_FTR_ALTIVEC_COMP | CPU_FTR_VSX_COMP | CPU_FTRS_POWER9 | \ + CPU_FTRS_POWER9_DD2_1 | CPU_FTRS_POWER9_DD2_2) #else #define CPU_FTRS_POSSIBLE \ (CPU_FTRS_PPC970 | CPU_FTRS_POWER5 | \ CPU_FTRS_POWER6 | CPU_FTRS_POWER7 | CPU_FTRS_POWER8E | \ - CPU_FTRS_POWER8 | CPU_FTRS_POWER8_DD1 | CPU_FTRS_CELL | \ - CPU_FTRS_PA6T | CPU_FTR_VSX_COMP | CPU_FTR_ALTIVEC_COMP | \ - CPU_FTRS_POWER9 | CPU_FTRS_POWER9_DD1 | CPU_FTRS_POWER9_DD2_1 | \ - CPU_FTRS_POWER9_DD2_2) + CPU_FTRS_POWER8 | CPU_FTRS_CELL | CPU_FTRS_PA6T | \ + CPU_FTR_VSX_COMP | CPU_FTR_ALTIVEC_COMP | CPU_FTRS_POWER9 | \ + CPU_FTRS_POWER9_DD2_1 | CPU_FTRS_POWER9_DD2_2) #endif /* CONFIG_CPU_LITTLE_ENDIAN */ #endif #else @@ -566,17 +559,15 @@ enum { #ifdef CONFIG_CPU_LITTLE_ENDIAN #define CPU_FTRS_ALWAYS \ (CPU_FTRS_POSSIBLE & ~CPU_FTR_HVMODE & CPU_FTRS_POWER7 & \ - CPU_FTRS_POWER8E & CPU_FTRS_POWER8 & CPU_FTRS_POWER8_DD1 & \ - CPU_FTRS_POWER9 & CPU_FTRS_POWER9_DD1 & CPU_FTRS_POWER9_DD2_1 & \ - CPU_FTRS_DT_CPU_BASE) + CPU_FTRS_POWER8E & CPU_FTRS_POWER8 & CPU_FTRS_POWER9 & \ + CPU_FTRS_POWER9_DD2_1 & CPU_FTRS_DT_CPU_BASE) #else #define CPU_FTRS_ALWAYS \ (CPU_FTRS_PPC970 & CPU_FTRS_POWER5 & \ CPU_FTRS_POWER6 & CPU_FTRS_POWER7 & CPU_FTRS_CELL & \ CPU_FTRS_PA6T & CPU_FTRS_POWER8 & CPU_FTRS_POWER8E & \ - CPU_FTRS_POWER8_DD1 & ~CPU_FTR_HVMODE & CPU_FTRS_POSSIBLE & \ - CPU_FTRS_POWER9 & CPU_FTRS_POWER9_DD1 & CPU_FTRS_POWER9_DD2_1 & \ - CPU_FTRS_DT_CPU_BASE) + ~CPU_FTR_HVMODE & CPU_FTRS_POSSIBLE & CPU_FTRS_POWER9 & \ + CPU_FTRS_POWER9_DD2_1 & CPU_FTRS_DT_CPU_BASE) #endif /* CONFIG_CPU_LITTLE_ENDIAN */ #endif #else diff --git a/arch/powerpc/include/asm/cputime.h b/arch/powerpc/include/asm/cputime.h index bc4903badb3f..133672744b2e 100644 --- a/arch/powerpc/include/asm/cputime.h +++ b/arch/powerpc/include/asm/cputime.h @@ -23,7 +23,6 @@ #include <asm/div64.h> #include <asm/time.h> #include <asm/param.h> -#include <asm/cpu_has_feature.h> typedef u64 __nocast cputime_t; typedef u64 __nocast cputime64_t; diff --git a/arch/powerpc/include/asm/dbell.h b/arch/powerpc/include/asm/dbell.h index 9f2ae0d25e15..99b84db23e8c 100644 --- a/arch/powerpc/include/asm/dbell.h +++ b/arch/powerpc/include/asm/dbell.h @@ -16,7 +16,7 @@ #include <linux/threads.h> #include <asm/ppc-opcode.h> -#include <asm/cpu_has_feature.h> +#include <asm/feature-fixups.h> #define PPC_DBELL_MSG_BRDCAST (0x04000000) #define PPC_DBELL_TYPE(x) (((x) & 0xf) << (63-36)) diff --git a/arch/powerpc/include/asm/dcr-native.h b/arch/powerpc/include/asm/dcr-native.h index 4a2beef74277..151dff555f50 100644 --- a/arch/powerpc/include/asm/dcr-native.h +++ b/arch/powerpc/include/asm/dcr-native.h @@ -25,6 +25,7 @@ #include <linux/spinlock.h> #include <asm/cputable.h> #include <asm/cpu_has_feature.h> +#include <linux/stringify.h> typedef struct { unsigned int base; diff --git a/arch/powerpc/include/asm/debug.h b/arch/powerpc/include/asm/debug.h index ce5da214ffe5..7756026b95ca 100644 --- a/arch/powerpc/include/asm/debug.h +++ b/arch/powerpc/include/asm/debug.h @@ -45,7 +45,6 @@ static inline int debugger_break_match(struct pt_regs *regs) { return 0; } static inline int debugger_fault_handler(struct pt_regs *regs) { return 0; } #endif -void set_breakpoint(struct arch_hw_breakpoint *brk); void __set_breakpoint(struct arch_hw_breakpoint *brk); bool ppc_breakpoint_available(void); #ifdef CONFIG_PPC_ADV_DEBUG_REGS diff --git a/arch/powerpc/include/asm/dt_cpu_ftrs.h b/arch/powerpc/include/asm/dt_cpu_ftrs.h index 71515d909ed1..0c729e2d0e8a 100644 --- a/arch/powerpc/include/asm/dt_cpu_ftrs.h +++ b/arch/powerpc/include/asm/dt_cpu_ftrs.h @@ -10,8 +10,6 @@ */ #include <linux/types.h> -#include <asm/asm-compat.h> -#include <asm/feature-fixups.h> #include <uapi/asm/cputable.h> #ifdef CONFIG_PPC_DT_CPU_FTRS diff --git a/arch/powerpc/include/asm/eeh.h b/arch/powerpc/include/asm/eeh.h index 677102baf3cd..219637ea69a1 100644 --- a/arch/powerpc/include/asm/eeh.h +++ b/arch/powerpc/include/asm/eeh.h @@ -36,13 +36,14 @@ struct pci_dn; #ifdef CONFIG_EEH /* EEH subsystem flags */ -#define EEH_ENABLED 0x01 /* EEH enabled */ -#define EEH_FORCE_DISABLED 0x02 /* EEH disabled */ -#define EEH_PROBE_MODE_DEV 0x04 /* From PCI device */ -#define EEH_PROBE_MODE_DEVTREE 0x08 /* From device tree */ -#define EEH_VALID_PE_ZERO 0x10 /* PE#0 is valid */ -#define EEH_ENABLE_IO_FOR_LOG 0x20 /* Enable IO for log */ -#define EEH_EARLY_DUMP_LOG 0x40 /* Dump log immediately */ +#define EEH_ENABLED 0x01 /* EEH enabled */ +#define EEH_FORCE_DISABLED 0x02 /* EEH disabled */ +#define EEH_PROBE_MODE_DEV 0x04 /* From PCI device */ +#define EEH_PROBE_MODE_DEVTREE 0x08 /* From device tree */ +#define EEH_VALID_PE_ZERO 0x10 /* PE#0 is valid */ +#define EEH_ENABLE_IO_FOR_LOG 0x20 /* Enable IO for log */ +#define EEH_EARLY_DUMP_LOG 0x40 /* Dump log immediately */ +#define EEH_POSTPONED_PROBE 0x80 /* Powernv may postpone device probe */ /* * Delay for PE reset, all in ms diff --git a/arch/powerpc/include/asm/exception-64s.h b/arch/powerpc/include/asm/exception-64s.h index c40b4380951c..a86feddddad0 100644 --- a/arch/powerpc/include/asm/exception-64s.h +++ b/arch/powerpc/include/asm/exception-64s.h @@ -35,6 +35,7 @@ * implementations as possible. */ #include <asm/head-64.h> +#include <asm/feature-fixups.h> /* PACA save area offsets (exgen, exmc, etc) */ #define EX_R9 0 @@ -156,7 +157,7 @@ b hrfi_flush_fallback #ifdef CONFIG_RELOCATABLE -#define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) \ +#define __EXCEPTION_PROLOG_2_RELON(label, h) \ mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ LOAD_HANDLER(r12,label); \ mtctr r12; \ @@ -166,25 +167,26 @@ bctr; #else /* If not relocatable, we can jump directly -- and save messing with LR */ -#define __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) \ +#define __EXCEPTION_PROLOG_2_RELON(label, h) \ mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ mfspr r12,SPRN_##h##SRR1; /* and SRR1 */ \ li r10,MSR_RI; \ mtmsrd r10,1; /* Set RI (EE=0) */ \ b label; #endif -#define EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) \ - __EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) \ +#define EXCEPTION_PROLOG_2_RELON(label, h) \ + __EXCEPTION_PROLOG_2_RELON(label, h) /* - * As EXCEPTION_PROLOG_PSERIES(), except we've already got relocation on - * so no need to rfid. Save lr in case we're CONFIG_RELOCATABLE, in which - * case EXCEPTION_RELON_PROLOG_PSERIES_1 will be using lr. + * As EXCEPTION_PROLOG(), except we've already got relocation on so no need to + * rfid. Save LR in case we're CONFIG_RELOCATABLE, in which case + * EXCEPTION_PROLOG_2_RELON will be using LR. */ -#define EXCEPTION_RELON_PROLOG_PSERIES(area, label, h, extra, vec) \ +#define EXCEPTION_RELON_PROLOG(area, label, h, extra, vec) \ + SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) + EXCEPTION_PROLOG_2_RELON(label, h) /* * We're short on space and time in the exception prolog, so we can't @@ -315,7 +317,7 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define EXCEPTION_PROLOG_1(area, extra, vec) \ _EXCEPTION_PROLOG_1(area, extra, vec) -#define __EXCEPTION_PROLOG_PSERIES_1(label, h) \ +#define __EXCEPTION_PROLOG_2(label, h) \ ld r10,PACAKMSR(r13); /* get MSR value for kernel */ \ mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ LOAD_HANDLER(r12,label) \ @@ -324,11 +326,11 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) mtspr SPRN_##h##SRR1,r10; \ h##RFI_TO_KERNEL; \ b . /* prevent speculative execution */ -#define EXCEPTION_PROLOG_PSERIES_1(label, h) \ - __EXCEPTION_PROLOG_PSERIES_1(label, h) +#define EXCEPTION_PROLOG_2(label, h) \ + __EXCEPTION_PROLOG_2(label, h) /* _NORI variant keeps MSR_RI clear */ -#define __EXCEPTION_PROLOG_PSERIES_1_NORI(label, h) \ +#define __EXCEPTION_PROLOG_2_NORI(label, h) \ ld r10,PACAKMSR(r13); /* get MSR value for kernel */ \ xori r10,r10,MSR_RI; /* Clear MSR_RI */ \ mfspr r11,SPRN_##h##SRR0; /* save SRR0 */ \ @@ -339,13 +341,14 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) h##RFI_TO_KERNEL; \ b . /* prevent speculative execution */ -#define EXCEPTION_PROLOG_PSERIES_1_NORI(label, h) \ - __EXCEPTION_PROLOG_PSERIES_1_NORI(label, h) +#define EXCEPTION_PROLOG_2_NORI(label, h) \ + __EXCEPTION_PROLOG_2_NORI(label, h) -#define EXCEPTION_PROLOG_PSERIES(area, label, h, extra, vec) \ +#define EXCEPTION_PROLOG(area, label, h, extra, vec) \ + SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_PSERIES_1(label, h); + EXCEPTION_PROLOG_2(label, h); #define __KVMTEST(h, n) \ lbz r10,HSTATE_IN_GUEST(r13); \ @@ -416,10 +419,10 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #endif /* Do not enable RI */ -#define EXCEPTION_PROLOG_PSERIES_NORI(area, label, h, extra, vec) \ +#define EXCEPTION_PROLOG_NORI(area, label, h, extra, vec) \ EXCEPTION_PROLOG_0(area); \ EXCEPTION_PROLOG_1(area, extra, vec); \ - EXCEPTION_PROLOG_PSERIES_1_NORI(label, h); + EXCEPTION_PROLOG_2_NORI(label, h); #define __KVM_HANDLER(area, h, n) \ @@ -550,10 +553,8 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) /* * Exception vectors. */ -#define STD_EXCEPTION_PSERIES(vec, label) \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label, \ - EXC_STD, KVMTEST_PR, vec); \ +#define STD_EXCEPTION(vec, label) \ + EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_STD, KVMTEST_PR, vec); /* Version of above for when we have to branch out-of-line */ #define __OOL_EXCEPTION(vec, label, hdlr) \ @@ -561,36 +562,31 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) EXCEPTION_PROLOG_0(PACA_EXGEN) \ b hdlr; -#define STD_EXCEPTION_PSERIES_OOL(vec, label) \ +#define STD_EXCEPTION_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_PR, vec); \ - EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD) + EXCEPTION_PROLOG_2(label, EXC_STD) #define STD_EXCEPTION_HV(loc, vec, label) \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_PROLOG_PSERIES(PACA_EXGEN, label, \ - EXC_HV, KVMTEST_HV, vec); + EXCEPTION_PROLOG(PACA_EXGEN, label, EXC_HV, KVMTEST_HV, vec); #define STD_EXCEPTION_HV_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \ - EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV) + EXCEPTION_PROLOG_2(label, EXC_HV) -#define STD_RELON_EXCEPTION_PSERIES(loc, vec, label) \ +#define STD_RELON_EXCEPTION(loc, vec, label) \ /* No guest interrupts come through here */ \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label, EXC_STD, NOTEST, vec); + EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_STD, NOTEST, vec); -#define STD_RELON_EXCEPTION_PSERIES_OOL(vec, label) \ +#define STD_RELON_EXCEPTION_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, NOTEST, vec); \ - EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_STD) + EXCEPTION_PROLOG_2_RELON(label, EXC_STD) #define STD_RELON_EXCEPTION_HV(loc, vec, label) \ - SET_SCRATCH0(r13); /* save r13 */ \ - EXCEPTION_RELON_PROLOG_PSERIES(PACA_EXGEN, label, \ - EXC_HV, KVMTEST_HV, vec); + EXCEPTION_RELON_PROLOG(PACA_EXGEN, label, EXC_HV, KVMTEST_HV, vec); #define STD_RELON_EXCEPTION_HV_OOL(vec, label) \ EXCEPTION_PROLOG_1(PACA_EXGEN, KVMTEST_HV, vec); \ - EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV) + EXCEPTION_PROLOG_2_RELON(label, EXC_HV) /* This associate vector numbers with bits in paca->irq_happened */ #define SOFTEN_VALUE_0x500 PACA_IRQ_EE @@ -627,55 +623,45 @@ END_FTR_SECTION_NESTED(ftr,ftr,943) #define SOFTEN_NOTEST_PR(vec, bitmask) _SOFTEN_TEST(EXC_STD, vec, bitmask) #define SOFTEN_NOTEST_HV(vec, bitmask) _SOFTEN_TEST(EXC_HV, vec, bitmask) -#define __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) \ +#define __MASKABLE_EXCEPTION(vec, label, h, extra, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ - EXCEPTION_PROLOG_PSERIES_1(label, h); + EXCEPTION_PROLOG_2(label, h); -#define _MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) \ - __MASKABLE_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) +#define MASKABLE_EXCEPTION(vec, label, bitmask) \ + __MASKABLE_EXCEPTION(vec, label, EXC_STD, SOFTEN_TEST_PR, bitmask) -#define MASKABLE_EXCEPTION_PSERIES(loc, vec, label, bitmask) \ - _MASKABLE_EXCEPTION_PSERIES(vec, label, \ - EXC_STD, SOFTEN_TEST_PR, bitmask) - -#define MASKABLE_EXCEPTION_PSERIES_OOL(vec, label, bitmask) \ +#define MASKABLE_EXCEPTION_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_PR, vec, bitmask);\ - EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD) + EXCEPTION_PROLOG_2(label, EXC_STD) -#define MASKABLE_EXCEPTION_HV(loc, vec, label, bitmask) \ - _MASKABLE_EXCEPTION_PSERIES(vec, label, \ - EXC_HV, SOFTEN_TEST_HV, bitmask) +#define MASKABLE_EXCEPTION_HV(vec, label, bitmask) \ + __MASKABLE_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) #define MASKABLE_EXCEPTION_HV_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ - EXCEPTION_PROLOG_PSERIES_1(label, EXC_HV) + EXCEPTION_PROLOG_2(label, EXC_HV) -#define __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) \ +#define __MASKABLE_RELON_EXCEPTION(vec, label, h, extra, bitmask) \ SET_SCRATCH0(r13); /* save r13 */ \ EXCEPTION_PROLOG_0(PACA_EXGEN); \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, extra, vec, bitmask); \ - EXCEPTION_RELON_PROLOG_PSERIES_1(label, h) - -#define _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask)\ - __MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, h, extra, bitmask) + EXCEPTION_PROLOG_2_RELON(label, h) -#define MASKABLE_RELON_EXCEPTION_PSERIES(loc, vec, label, bitmask) \ - _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \ - EXC_STD, SOFTEN_NOTEST_PR, bitmask) +#define MASKABLE_RELON_EXCEPTION(vec, label, bitmask) \ + __MASKABLE_RELON_EXCEPTION(vec, label, EXC_STD, SOFTEN_NOTEST_PR, bitmask) -#define MASKABLE_RELON_EXCEPTION_PSERIES_OOL(vec, label, bitmask) \ +#define MASKABLE_RELON_EXCEPTION_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_NOTEST_PR, vec, bitmask);\ - EXCEPTION_PROLOG_PSERIES_1(label, EXC_STD); + EXCEPTION_PROLOG_2(label, EXC_STD); -#define MASKABLE_RELON_EXCEPTION_HV(loc, vec, label, bitmask) \ - _MASKABLE_RELON_EXCEPTION_PSERIES(vec, label, \ - EXC_HV, SOFTEN_TEST_HV, bitmask) +#define MASKABLE_RELON_EXCEPTION_HV(vec, label, bitmask) \ + __MASKABLE_RELON_EXCEPTION(vec, label, EXC_HV, SOFTEN_TEST_HV, bitmask) #define MASKABLE_RELON_EXCEPTION_HV_OOL(vec, label, bitmask) \ MASKABLE_EXCEPTION_PROLOG_1(PACA_EXGEN, SOFTEN_TEST_HV, vec, bitmask);\ - EXCEPTION_RELON_PROLOG_PSERIES_1(label, EXC_HV) + EXCEPTION_PROLOG_2_RELON(label, EXC_HV) /* * Our exception common code can be passed various "additions" diff --git a/arch/powerpc/include/asm/fadump.h b/arch/powerpc/include/asm/fadump.h index 5a23010af600..1e7a33592e29 100644 --- a/arch/powerpc/include/asm/fadump.h +++ b/arch/powerpc/include/asm/fadump.h @@ -195,9 +195,6 @@ struct fadump_crash_info_header { struct cpumask online_mask; }; -/* Crash memory ranges */ -#define INIT_CRASHMEM_RANGES (INIT_MEMBLOCK_REGIONS + 2) - struct fad_crash_memory_ranges { unsigned long long base; unsigned long long size; diff --git a/arch/powerpc/include/asm/feature-fixups.h b/arch/powerpc/include/asm/feature-fixups.h index fcfd05672b1b..33b6f9c892c8 100644 --- a/arch/powerpc/include/asm/feature-fixups.h +++ b/arch/powerpc/include/asm/feature-fixups.h @@ -1,6 +1,8 @@ #ifndef __ASM_POWERPC_FEATURE_FIXUPS_H #define __ASM_POWERPC_FEATURE_FIXUPS_H +#include <asm/asm-const.h> + /* * This program is free software; you can redistribute it and/or * modify it under the terms of the GNU General Public License diff --git a/arch/powerpc/include/asm/firmware.h b/arch/powerpc/include/asm/firmware.h index 535add3f7791..7a051bd21f87 100644 --- a/arch/powerpc/include/asm/firmware.h +++ b/arch/powerpc/include/asm/firmware.h @@ -14,8 +14,7 @@ #ifdef __KERNEL__ -#include <asm/asm-compat.h> -#include <asm/feature-fixups.h> +#include <asm/asm-const.h> /* firmware feature bitmask values */ diff --git a/arch/powerpc/include/asm/fixmap.h b/arch/powerpc/include/asm/fixmap.h index 6c40dfda5912..41cc15c14eee 100644 --- a/arch/powerpc/include/asm/fixmap.h +++ b/arch/powerpc/include/asm/fixmap.h @@ -15,7 +15,6 @@ #define _ASM_FIXMAP_H #ifndef __ASSEMBLY__ -#include <linux/kernel.h> #include <asm/page.h> #include <asm/pgtable.h> #ifdef CONFIG_HIGHMEM diff --git a/arch/powerpc/include/asm/futex.h b/arch/powerpc/include/asm/futex.h index 1a944c18c539..94542776a62d 100644 --- a/arch/powerpc/include/asm/futex.h +++ b/arch/powerpc/include/asm/futex.h @@ -8,7 +8,7 @@ #include <linux/uaccess.h> #include <asm/errno.h> #include <asm/synch.h> -#include <asm/asm-compat.h> +#include <asm/asm-405.h> #define __futex_atomic_op(insn, ret, oldval, uaddr, oparg) \ __asm__ __volatile ( \ diff --git a/arch/powerpc/include/asm/head-64.h b/arch/powerpc/include/asm/head-64.h index 7e0e93f24cb7..a4f947888744 100644 --- a/arch/powerpc/include/asm/head-64.h +++ b/arch/powerpc/include/asm/head-64.h @@ -260,22 +260,22 @@ name: #define EXC_REAL(name, start, size) \ EXC_REAL_BEGIN(name, start, size); \ - STD_EXCEPTION_PSERIES(start, name##_common); \ + STD_EXCEPTION(start, name##_common); \ EXC_REAL_END(name, start, size); #define EXC_VIRT(name, start, size, realvec) \ EXC_VIRT_BEGIN(name, start, size); \ - STD_RELON_EXCEPTION_PSERIES(start, realvec, name##_common); \ + STD_RELON_EXCEPTION(start, realvec, name##_common); \ EXC_VIRT_END(name, start, size); #define EXC_REAL_MASKABLE(name, start, size, bitmask) \ EXC_REAL_BEGIN(name, start, size); \ - MASKABLE_EXCEPTION_PSERIES(start, start, name##_common, bitmask);\ + MASKABLE_EXCEPTION(start, name##_common, bitmask); \ EXC_REAL_END(name, start, size); #define EXC_VIRT_MASKABLE(name, start, size, realvec, bitmask) \ EXC_VIRT_BEGIN(name, start, size); \ - MASKABLE_RELON_EXCEPTION_PSERIES(start, realvec, name##_common, bitmask);\ + MASKABLE_RELON_EXCEPTION(realvec, name##_common, bitmask); \ EXC_VIRT_END(name, start, size); #define EXC_REAL_HV(name, start, size) \ @@ -295,7 +295,7 @@ name: #define __TRAMP_REAL_OOL(name, vec) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - STD_EXCEPTION_PSERIES_OOL(vec, name##_common); \ + STD_EXCEPTION_OOL(vec, name##_common); #define EXC_REAL_OOL(name, start, size) \ __EXC_REAL_OOL(name, start, size); \ @@ -306,7 +306,7 @@ name: #define __TRAMP_REAL_OOL_MASKABLE(name, vec, bitmask) \ TRAMP_REAL_BEGIN(tramp_real_##name); \ - MASKABLE_EXCEPTION_PSERIES_OOL(vec, name##_common, bitmask); \ + MASKABLE_EXCEPTION_OOL(vec, name##_common, bitmask); #define EXC_REAL_OOL_MASKABLE(name, start, size, bitmask) \ __EXC_REAL_OOL_MASKABLE(name, start, size); \ @@ -346,7 +346,7 @@ name: #define __TRAMP_VIRT_OOL(name, realvec) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - STD_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common); \ + STD_RELON_EXCEPTION_OOL(realvec, name##_common); #define EXC_VIRT_OOL(name, start, size, realvec) \ __EXC_VIRT_OOL(name, start, size); \ @@ -357,7 +357,7 @@ name: #define __TRAMP_VIRT_OOL_MASKABLE(name, realvec, bitmask) \ TRAMP_VIRT_BEGIN(tramp_virt_##name); \ - MASKABLE_RELON_EXCEPTION_PSERIES_OOL(realvec, name##_common, bitmask);\ + MASKABLE_RELON_EXCEPTION_OOL(realvec, name##_common, bitmask); #define EXC_VIRT_OOL_MASKABLE(name, start, size, realvec, bitmask) \ __EXC_VIRT_OOL_MASKABLE(name, start, size); \ diff --git a/arch/powerpc/include/asm/highmem.h b/arch/powerpc/include/asm/highmem.h index cec820f961da..a4b65b186ec6 100644 --- a/arch/powerpc/include/asm/highmem.h +++ b/arch/powerpc/include/asm/highmem.h @@ -25,7 +25,7 @@ #include <linux/interrupt.h> #include <asm/kmap_types.h> -#include <asm/tlbflush.h> +#include <asm/cacheflush.h> #include <asm/page.h> #include <asm/fixmap.h> diff --git a/arch/powerpc/include/asm/hugetlb.h b/arch/powerpc/include/asm/hugetlb.h index 3225eb6402cc..2d00cc530083 100644 --- a/arch/powerpc/include/asm/hugetlb.h +++ b/arch/powerpc/include/asm/hugetlb.h @@ -84,9 +84,6 @@ static inline pte_t *hugepte_offset(hugepd_t hpd, unsigned long addr, return dir + idx; } -pte_t *huge_pte_offset_and_shift(struct mm_struct *mm, - unsigned long addr, unsigned *shift); - void flush_dcache_icache_hugepage(struct page *page); int slice_is_hugepage_only_range(struct mm_struct *mm, unsigned long addr, diff --git a/arch/powerpc/include/asm/hvcall.h b/arch/powerpc/include/asm/hvcall.h index 662c8347d699..a0b17f9f1ea4 100644 --- a/arch/powerpc/include/asm/hvcall.h +++ b/arch/powerpc/include/asm/hvcall.h @@ -342,10 +342,12 @@ #define H_CPU_CHAR_BRANCH_HINTS_HONORED (1ull << 58) // IBM bit 5 #define H_CPU_CHAR_THREAD_RECONFIG_CTRL (1ull << 57) // IBM bit 6 #define H_CPU_CHAR_COUNT_CACHE_DISABLED (1ull << 56) // IBM bit 7 +#define H_CPU_CHAR_BCCTR_FLUSH_ASSIST (1ull << 54) // IBM bit 9 #define H_CPU_BEHAV_FAVOUR_SECURITY (1ull << 63) // IBM bit 0 #define H_CPU_BEHAV_L1D_FLUSH_PR (1ull << 62) // IBM bit 1 #define H_CPU_BEHAV_BNDS_CHK_SPEC_BAR (1ull << 61) // IBM bit 2 +#define H_CPU_BEHAV_FLUSH_COUNT_CACHE (1ull << 58) // IBM bit 5 /* Flag values used in H_REGISTER_PROC_TBL hcall */ #define PROC_TABLE_OP_MASK 0x18 diff --git a/arch/powerpc/include/asm/hw_breakpoint.h b/arch/powerpc/include/asm/hw_breakpoint.h index 27d6e3c8fde9..ece4dc89c90b 100644 --- a/arch/powerpc/include/asm/hw_breakpoint.h +++ b/arch/powerpc/include/asm/hw_breakpoint.h @@ -56,6 +56,7 @@ struct perf_event_attr; struct perf_event; struct pmu; struct perf_sample_data; +struct task_struct; #define HW_BREAKPOINT_ALIGN 0x7 diff --git a/arch/powerpc/include/asm/hw_irq.h b/arch/powerpc/include/asm/hw_irq.h index e151774cb577..32a18f2f49bc 100644 --- a/arch/powerpc/include/asm/hw_irq.h +++ b/arch/powerpc/include/asm/hw_irq.h @@ -253,14 +253,16 @@ static inline bool lazy_irq_pending(void) /* * This is called by asynchronous interrupts to conditionally - * re-enable hard interrupts when soft-disabled after having - * cleared the source of the interrupt + * re-enable hard interrupts after having cleared the source + * of the interrupt. They are kept disabled if there is a different + * soft-masked interrupt pending that requires hard masking. */ static inline void may_hard_irq_enable(void) { - get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS; - if (!(get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK)) + if (!(get_paca()->irq_happened & PACA_IRQ_MUST_HARD_MASK)) { + get_paca()->irq_happened &= ~PACA_IRQ_HARD_DIS; __hard_irq_enable(); + } } static inline bool arch_irq_disabled_regs(struct pt_regs *regs) diff --git a/arch/powerpc/include/asm/iommu.h b/arch/powerpc/include/asm/iommu.h index 20febe0b7f32..ab3a4fba38e3 100644 --- a/arch/powerpc/include/asm/iommu.h +++ b/arch/powerpc/include/asm/iommu.h @@ -30,6 +30,7 @@ #include <asm/machdep.h> #include <asm/types.h> #include <asm/pci-bridge.h> +#include <asm/asm-const.h> #define IOMMU_PAGE_SHIFT_4K 12 #define IOMMU_PAGE_SIZE_4K (ASM_CONST(1) << IOMMU_PAGE_SHIFT_4K) @@ -69,6 +70,8 @@ struct iommu_table_ops { long index, unsigned long *hpa, enum dma_data_direction *direction); + + __be64 *(*useraddrptr)(struct iommu_table *tbl, long index, bool alloc); #endif void (*clear)(struct iommu_table *tbl, long index, long npages); @@ -117,15 +120,16 @@ struct iommu_table { unsigned long *it_map; /* A simple allocation bitmap for now */ unsigned long it_page_shift;/* table iommu page size */ struct list_head it_group_list;/* List of iommu_table_group_link */ - unsigned long *it_userspace; /* userspace view of the table */ + __be64 *it_userspace; /* userspace view of the table */ struct iommu_table_ops *it_ops; struct kref it_kref; + int it_nid; }; +#define IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry) \ + ((tbl)->it_ops->useraddrptr((tbl), (entry), false)) #define IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry) \ - ((tbl)->it_userspace ? \ - &((tbl)->it_userspace[(entry) - (tbl)->it_offset]) : \ - NULL) + ((tbl)->it_ops->useraddrptr((tbl), (entry), true)) /* Pure 2^n version of get_order */ static inline __attribute_const__ diff --git a/arch/powerpc/include/asm/jump_label.h b/arch/powerpc/include/asm/jump_label.h index 9a287e0ac8b1..a3b2cf940b4e 100644 --- a/arch/powerpc/include/asm/jump_label.h +++ b/arch/powerpc/include/asm/jump_label.h @@ -14,7 +14,7 @@ #include <linux/types.h> #include <asm/feature-fixups.h> -#include <asm/asm-compat.h> +#include <asm/asm-const.h> #define JUMP_ENTRY_TYPE stringify_in_c(FTR_ENTRY_LONG) #define JUMP_LABEL_NOP_SIZE 4 diff --git a/arch/powerpc/include/asm/kvm_booke_hv_asm.h b/arch/powerpc/include/asm/kvm_booke_hv_asm.h index e5f048bbcb7c..931260b59ac6 100644 --- a/arch/powerpc/include/asm/kvm_booke_hv_asm.h +++ b/arch/powerpc/include/asm/kvm_booke_hv_asm.h @@ -9,6 +9,8 @@ #ifndef ASM_KVM_BOOKE_HV_ASM_H #define ASM_KVM_BOOKE_HV_ASM_H +#include <asm/feature-fixups.h> + #ifdef __ASSEMBLY__ /* diff --git a/arch/powerpc/include/asm/mmu-44x.h b/arch/powerpc/include/asm/mmu-44x.h index cb57f29f531d..295b3dbb2698 100644 --- a/arch/powerpc/include/asm/mmu-44x.h +++ b/arch/powerpc/include/asm/mmu-44x.h @@ -5,7 +5,7 @@ * PPC440 support */ -#include <asm/page.h> +#include <asm/asm-const.h> #define PPC44x_MMUCR_TID 0x000000ff #define PPC44x_MMUCR_STS 0x00010000 @@ -124,19 +124,19 @@ typedef struct { /* Size of the TLBs used for pinning in lowmem */ #define PPC_PIN_SIZE (1 << 28) /* 256M */ -#if (PAGE_SHIFT == 12) +#if defined(CONFIG_PPC_4K_PAGES) #define PPC44x_TLBE_SIZE PPC44x_TLB_4K #define PPC47x_TLBE_SIZE PPC47x_TLB0_4K #define mmu_virtual_psize MMU_PAGE_4K -#elif (PAGE_SHIFT == 14) +#elif defined(CONFIG_PPC_16K_PAGES) #define PPC44x_TLBE_SIZE PPC44x_TLB_16K #define PPC47x_TLBE_SIZE PPC47x_TLB0_16K #define mmu_virtual_psize MMU_PAGE_16K -#elif (PAGE_SHIFT == 16) +#elif defined(CONFIG_PPC_64K_PAGES) #define PPC44x_TLBE_SIZE PPC44x_TLB_64K #define PPC47x_TLBE_SIZE PPC47x_TLB0_64K #define mmu_virtual_psize MMU_PAGE_64K -#elif (PAGE_SHIFT == 18) +#elif defined(CONFIG_PPC_256K_PAGES) #define PPC44x_TLBE_SIZE PPC44x_TLB_256K #define mmu_virtual_psize MMU_PAGE_256K #else diff --git a/arch/powerpc/include/asm/mmu.h b/arch/powerpc/include/asm/mmu.h index 61d15ce92278..13ea441ac531 100644 --- a/arch/powerpc/include/asm/mmu.h +++ b/arch/powerpc/include/asm/mmu.h @@ -5,8 +5,7 @@ #include <linux/types.h> -#include <asm/asm-compat.h> -#include <asm/feature-fixups.h> +#include <asm/asm-const.h> /* * MMU features bit definitions diff --git a/arch/powerpc/include/asm/nohash/32/pgtable.h b/arch/powerpc/include/asm/nohash/32/pgtable.h index 7c46a98cc7f4..a507a65b0866 100644 --- a/arch/powerpc/include/asm/nohash/32/pgtable.h +++ b/arch/powerpc/include/asm/nohash/32/pgtable.h @@ -8,7 +8,8 @@ #ifndef __ASSEMBLY__ #include <linux/sched.h> #include <linux/threads.h> -#include <asm/io.h> /* For sub-arch specific PPC_PIN_SIZE */ +#include <asm/mmu.h> /* For sub-arch specific PPC_PIN_SIZE */ +#include <asm/asm-405.h> extern unsigned long ioremap_bot; @@ -222,10 +223,6 @@ static inline unsigned long long pte_update(pte_t *p, } #endif /* CONFIG_PTE_64BIT */ -/* - * 2.6 calls this without flushing the TLB entry; this is wrong - * for our hash-based implementation, we fix that up here. - */ #define __HAVE_ARCH_PTEP_TEST_AND_CLEAR_YOUNG static inline int __ptep_test_and_clear_young(unsigned int context, unsigned long addr, pte_t *ptep) { diff --git a/arch/powerpc/include/asm/nohash/64/pgtable.h b/arch/powerpc/include/asm/nohash/64/pgtable.h index dd0c7236208f..7cd6809f4d33 100644 --- a/arch/powerpc/include/asm/nohash/64/pgtable.h +++ b/arch/powerpc/include/asm/nohash/64/pgtable.h @@ -3,11 +3,12 @@ #define _ASM_POWERPC_NOHASH_64_PGTABLE_H /* * This file contains the functions and defines necessary to modify and use - * the ppc64 hashed page table. + * the ppc64 non-hashed page table. */ #include <asm/nohash/64/pgtable-4k.h> #include <asm/barrier.h> +#include <asm/asm-const.h> #ifdef CONFIG_PPC_64K_PAGES #error "Page size not supported" @@ -37,7 +38,7 @@ /* * The vmalloc space starts at the beginning of that region, and - * occupies half of it on hash CPUs and a quarter of it on Book3E + * occupies a quarter of it on Book3E * (we keep a quarter for the virtual memmap) */ #define VMALLOC_START KERN_VIRT_START @@ -77,7 +78,7 @@ /* * Defines the address of the vmemap area, in its own region on - * hash table CPUs and after the vmalloc space on Book3E + * after the vmalloc space on Book3E */ #define VMEMMAP_BASE VMALLOC_END #define VMEMMAP_END KERN_IO_START @@ -247,14 +248,6 @@ static inline void huge_ptep_set_wrprotect(struct mm_struct *mm, pte_update(mm, addr, ptep, _PAGE_RW, 0, 1); } -/* - * We currently remove entries from the hashtable regardless of whether - * the entry was young or dirty. The generic routines only flush if the - * entry was young or dirty which is not good enough. - * - * We should be more intelligent about this but for the moment we override - * these functions and force a tlb flush unconditionally - */ #define __HAVE_ARCH_PTEP_CLEAR_YOUNG_FLUSH #define ptep_clear_flush_young(__vma, __address, __ptep) \ ({ \ @@ -278,9 +271,7 @@ static inline void pte_clear(struct mm_struct *mm, unsigned long addr, } -/* Set the dirty and/or accessed bits atomically in a linux PTE, this - * function doesn't need to flush the hash entry - */ +/* Set the dirty and/or accessed bits atomically in a linux PTE */ static inline void __ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, pte_t entry, unsigned long address, diff --git a/arch/powerpc/include/asm/nohash/tlbflush.h b/arch/powerpc/include/asm/nohash/tlbflush.h new file mode 100644 index 000000000000..b1d8fec29169 --- /dev/null +++ b/arch/powerpc/include/asm/nohash/tlbflush.h @@ -0,0 +1,53 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _ASM_POWERPC_NOHASH_TLBFLUSH_H +#define _ASM_POWERPC_NOHASH_TLBFLUSH_H + +/* + * TLB flushing: + * + * - flush_tlb_mm(mm) flushes the specified mm context TLB's + * - flush_tlb_page(vma, vmaddr) flushes one page + * - local_flush_tlb_mm(mm, full) flushes the specified mm context on + * the local processor + * - local_flush_tlb_page(vma, vmaddr) flushes one page on the local processor + * - flush_tlb_page_nohash(vma, vmaddr) flushes one page if SW loaded TLB + * - flush_tlb_range(vma, start, end) flushes a range of pages + * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages + * + */ + +/* + * TLB flushing for software loaded TLB chips + * + * TODO: (CONFIG_FSL_BOOKE) determine if flush_tlb_range & + * flush_tlb_kernel_range are best implemented as tlbia vs + * specific tlbie's + */ + +struct vm_area_struct; +struct mm_struct; + +#define MMU_NO_CONTEXT ((unsigned int)-1) + +extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, + unsigned long end); +extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); + +extern void local_flush_tlb_mm(struct mm_struct *mm); +extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); + +extern void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, + int tsize, int ind); + +#ifdef CONFIG_SMP +extern void flush_tlb_mm(struct mm_struct *mm); +extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); +extern void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, + int tsize, int ind); +#else +#define flush_tlb_mm(mm) local_flush_tlb_mm(mm) +#define flush_tlb_page(vma,addr) local_flush_tlb_page(vma,addr) +#define __flush_tlb_page(mm,addr,p,i) __local_flush_tlb_page(mm,addr,p,i) +#endif + +#endif /* _ASM_POWERPC_NOHASH_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/opal-api.h b/arch/powerpc/include/asm/opal-api.h index 3bab299eda49..8365353330b4 100644 --- a/arch/powerpc/include/asm/opal-api.h +++ b/arch/powerpc/include/asm/opal-api.h @@ -206,9 +206,11 @@ #define OPAL_NPU_SPA_CLEAR_CACHE 160 #define OPAL_NPU_TL_SET 161 #define OPAL_SENSOR_READ_U64 162 +#define OPAL_SENSOR_GROUP_ENABLE 163 #define OPAL_PCI_GET_PBCQ_TUNNEL_BAR 164 #define OPAL_PCI_SET_PBCQ_TUNNEL_BAR 165 -#define OPAL_LAST 165 +#define OPAL_NX_COPROC_INIT 167 +#define OPAL_LAST 167 #define QUIESCE_HOLD 1 /* Spin all calls at entry */ #define QUIESCE_REJECT 2 /* Fail all calls with OPAL_BUSY */ diff --git a/arch/powerpc/include/asm/opal.h b/arch/powerpc/include/asm/opal.h index e1b2910c6e81..834e7e29f1e4 100644 --- a/arch/powerpc/include/asm/opal.h +++ b/arch/powerpc/include/asm/opal.h @@ -292,6 +292,8 @@ int opal_set_powercap(u32 handle, int token, u32 pcap); int opal_get_power_shift_ratio(u32 handle, int token, u32 *psr); int opal_set_power_shift_ratio(u32 handle, int token, u32 psr); int opal_sensor_group_clear(u32 group_hndl, int token); +int opal_sensor_group_enable(u32 group_hndl, int token, bool enable); +int opal_nx_coproc_init(uint32_t chip_id, uint32_t ct); s64 opal_signal_system_reset(s32 cpu); s64 opal_quiesce(u64 shutdown_type, s32 cpu); @@ -305,6 +307,8 @@ extern void opal_configure_cores(void); extern int opal_get_chars(uint32_t vtermno, char *buf, int count); extern int opal_put_chars(uint32_t vtermno, const char *buf, int total_len); +extern int opal_put_chars_atomic(uint32_t vtermno, const char *buf, int total_len); +extern int opal_flush_console(uint32_t vtermno); extern void hvc_opal_init_early(void); @@ -326,6 +330,7 @@ extern int opal_async_wait_response_interruptible(uint64_t token, struct opal_msg *msg); extern int opal_get_sensor_data(u32 sensor_hndl, u32 *sensor_data); extern int opal_get_sensor_data_u64(u32 sensor_hndl, u64 *sensor_data); +extern int sensor_group_enable(u32 grp_hndl, bool enable); struct rtc_time; extern time64_t opal_get_boot_time(void); diff --git a/arch/powerpc/include/asm/paca.h b/arch/powerpc/include/asm/paca.h index 6d34bd71139d..ad4f16164619 100644 --- a/arch/powerpc/include/asm/paca.h +++ b/arch/powerpc/include/asm/paca.h @@ -187,11 +187,6 @@ struct paca_struct { u8 subcore_sibling_mask; /* Flag to request this thread not to stop */ atomic_t dont_stop; - /* - * Pointer to an array which contains pointer - * to the sibling threads' paca. - */ - struct paca_struct **thread_sibling_pacas; /* The PSSCR value that the kernel requested before going to stop */ u64 requested_psscr; @@ -252,6 +247,9 @@ struct paca_struct { void *rfi_flush_fallback_area; u64 l1d_flush_size; #endif +#ifdef CONFIG_PPC_PSERIES + u8 *mce_data_buf; /* buffer to hold per cpu rtas errlog */ +#endif /* CONFIG_PPC_PSERIES */ } ____cacheline_aligned; extern void copy_mm_to_paca(struct mm_struct *mm); diff --git a/arch/powerpc/include/asm/page.h b/arch/powerpc/include/asm/page.h index db7be0779d55..f6a1265face2 100644 --- a/arch/powerpc/include/asm/page.h +++ b/arch/powerpc/include/asm/page.h @@ -16,8 +16,7 @@ #else #include <asm/types.h> #endif -#include <asm/asm-compat.h> -#include <asm/kdump.h> +#include <asm/asm-const.h> /* * On regular PPC32 page size is 4K (but we support 4K/16K/64K/256K pages diff --git a/arch/powerpc/include/asm/page_64.h b/arch/powerpc/include/asm/page_64.h index af04acdb873f..c0ce17e909ef 100644 --- a/arch/powerpc/include/asm/page_64.h +++ b/arch/powerpc/include/asm/page_64.h @@ -10,6 +10,8 @@ * 2 of the License, or (at your option) any later version. */ +#include <asm/asm-const.h> + /* * We always define HW_PAGE_SHIFT to 12 as use of 64K pages remains Linux * specific, every notion of page number shared with the firmware, TCEs, diff --git a/arch/powerpc/include/asm/pkeys.h b/arch/powerpc/include/asm/pkeys.h index 5ba80cffb505..20ebf153c871 100644 --- a/arch/powerpc/include/asm/pkeys.h +++ b/arch/powerpc/include/asm/pkeys.h @@ -13,7 +13,8 @@ DECLARE_STATIC_KEY_TRUE(pkey_disabled); extern int pkeys_total; /* total pkeys as per device tree */ -extern u32 initial_allocation_mask; /* bits set for reserved keys */ +extern u32 initial_allocation_mask; /* bits set for the initially allocated keys */ +extern u32 reserved_allocation_mask; /* bits set for reserved keys */ #define ARCH_VM_PKEY_FLAGS (VM_PKEY_BIT0 | VM_PKEY_BIT1 | VM_PKEY_BIT2 | \ VM_PKEY_BIT3 | VM_PKEY_BIT4) @@ -83,19 +84,21 @@ static inline u16 pte_to_pkey_bits(u64 pteflags) #define __mm_pkey_is_allocated(mm, pkey) \ (mm_pkey_allocation_map(mm) & pkey_alloc_mask(pkey)) -#define __mm_pkey_is_reserved(pkey) (initial_allocation_mask & \ +#define __mm_pkey_is_reserved(pkey) (reserved_allocation_mask & \ pkey_alloc_mask(pkey)) static inline bool mm_pkey_is_allocated(struct mm_struct *mm, int pkey) { - /* A reserved key is never considered as 'explicitly allocated' */ - return ((pkey < arch_max_pkey()) && - !__mm_pkey_is_reserved(pkey) && - __mm_pkey_is_allocated(mm, pkey)); + if (pkey < 0 || pkey >= arch_max_pkey()) + return false; + + /* Reserved keys are never allocated. */ + if (__mm_pkey_is_reserved(pkey)) + return false; + + return __mm_pkey_is_allocated(mm, pkey); } -extern void __arch_activate_pkey(int pkey); -extern void __arch_deactivate_pkey(int pkey); /* * Returns a positive, 5-bit key on success, or -1 on failure. * Relies on the mmap_sem to protect against concurrency in mm_pkey_alloc() and @@ -124,11 +127,6 @@ static inline int mm_pkey_alloc(struct mm_struct *mm) ret = ffz((u32)mm_pkey_allocation_map(mm)); __mm_pkey_allocated(mm, ret); - /* - * Enable the key in the hardware - */ - if (ret > 0) - __arch_activate_pkey(ret); return ret; } @@ -140,10 +138,6 @@ static inline int mm_pkey_free(struct mm_struct *mm, int pkey) if (!mm_pkey_is_allocated(mm, pkey)) return -EINVAL; - /* - * Disable the key in the hardware - */ - __arch_deactivate_pkey(pkey); __mm_pkey_free(mm, pkey); return 0; @@ -187,6 +181,16 @@ static inline int arch_set_user_pkey_access(struct task_struct *tsk, int pkey, { if (static_branch_likely(&pkey_disabled)) return -EINVAL; + + /* + * userspace should not change pkey-0 permissions. + * pkey-0 is associated with every page in the kernel. + * If userspace denies any permission on pkey-0, the + * kernel cannot operate. + */ + if (pkey == 0) + return init_val ? -EINVAL : 0; + return __arch_set_user_pkey_access(tsk, pkey, init_val); } diff --git a/arch/powerpc/include/asm/pnv-pci.h b/arch/powerpc/include/asm/pnv-pci.h index d2d8c28db336..7f627e3f4da4 100644 --- a/arch/powerpc/include/asm/pnv-pci.h +++ b/arch/powerpc/include/asm/pnv-pci.h @@ -50,13 +50,6 @@ int pnv_cxl_alloc_hwirq_ranges(struct cxl_irq_ranges *irqs, struct pci_dev *dev, int num); void pnv_cxl_release_hwirq_ranges(struct cxl_irq_ranges *irqs, struct pci_dev *dev); - -/* Support for the cxl kernel api on the real PHB (instead of vPHB) */ -int pnv_cxl_enable_phb_kernel_api(struct pci_controller *hose, bool enable); -bool pnv_pci_on_cxl_phb(struct pci_dev *dev); -struct cxl_afu *pnv_cxl_phb_to_afu(struct pci_controller *hose); -void pnv_cxl_phb_set_peer_afu(struct pci_dev *dev, struct cxl_afu *afu); - #endif struct pnv_php_slot { diff --git a/arch/powerpc/include/asm/ppc-opcode.h b/arch/powerpc/include/asm/ppc-opcode.h index 4436887bc415..665af14850e4 100644 --- a/arch/powerpc/include/asm/ppc-opcode.h +++ b/arch/powerpc/include/asm/ppc-opcode.h @@ -12,8 +12,7 @@ #ifndef _ASM_POWERPC_PPC_OPCODE_H #define _ASM_POWERPC_PPC_OPCODE_H -#include <linux/stringify.h> -#include <asm/asm-compat.h> +#include <asm/asm-const.h> #define __REG_R0 0 #define __REG_R1 1 @@ -367,6 +366,8 @@ #define PPC_INST_STFDX 0x7c0005ae #define PPC_INST_LVX 0x7c0000ce #define PPC_INST_STVX 0x7c0001ce +#define PPC_INST_VCMPEQUD 0x100000c7 +#define PPC_INST_VCMPEQUB 0x10000006 /* macros to insert fields into opcodes */ #define ___PPC_RA(a) (((a) & 0x1f) << 16) @@ -397,6 +398,7 @@ #define __PPC_BI(s) (((s) & 0x1f) << 16) #define __PPC_CT(t) (((t) & 0x0f) << 21) #define __PPC_SPR(r) ((((r) & 0x1f) << 16) | ((((r) >> 5) & 0x1f) << 11)) +#define __PPC_RC21 (0x1 << 10) /* * Only use the larx hint bit on 64bit CPUs. e500v1/v2 based CPUs will treat a @@ -568,4 +570,12 @@ ((IH & 0x7) << 21)) #define PPC_INVALIDATE_ERAT PPC_SLBIA(7) +#define VCMPEQUD_RC(vrt, vra, vrb) stringify_in_c(.long PPC_INST_VCMPEQUD | \ + ___PPC_RT(vrt) | ___PPC_RA(vra) | \ + ___PPC_RB(vrb) | __PPC_RC21) + +#define VCMPEQUB_RC(vrt, vra, vrb) stringify_in_c(.long PPC_INST_VCMPEQUB | \ + ___PPC_RT(vrt) | ___PPC_RA(vra) | \ + ___PPC_RB(vrb) | __PPC_RC21) + #endif /* _ASM_POWERPC_PPC_OPCODE_H */ diff --git a/arch/powerpc/include/asm/ppc_asm.h b/arch/powerpc/include/asm/ppc_asm.h index 75ece56dcd62..b5d023680801 100644 --- a/arch/powerpc/include/asm/ppc_asm.h +++ b/arch/powerpc/include/asm/ppc_asm.h @@ -9,6 +9,7 @@ #include <asm/processor.h> #include <asm/ppc-opcode.h> #include <asm/firmware.h> +#include <asm/feature-fixups.h> #ifdef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/processor.h b/arch/powerpc/include/asm/processor.h index 5debe337ea9d..52fadded5c1e 100644 --- a/arch/powerpc/include/asm/processor.h +++ b/arch/powerpc/include/asm/processor.h @@ -39,10 +39,9 @@ #endif /* CONFIG_PPC64 */ #ifndef __ASSEMBLY__ -#include <linux/compiler.h> -#include <linux/cache.h> +#include <linux/types.h> +#include <asm/thread_info.h> #include <asm/ptrace.h> -#include <asm/types.h> #include <asm/hw_breakpoint.h> /* We do _not_ want to define new machine types at all, those must die diff --git a/arch/powerpc/include/asm/ptrace.h b/arch/powerpc/include/asm/ptrace.h index e4923686e43a..447cbd1bee99 100644 --- a/arch/powerpc/include/asm/ptrace.h +++ b/arch/powerpc/include/asm/ptrace.h @@ -24,6 +24,7 @@ #define _ASM_POWERPC_PTRACE_H #include <uapi/asm/ptrace.h> +#include <asm/asm-const.h> #ifdef __powerpc64__ diff --git a/arch/powerpc/include/asm/reg.h b/arch/powerpc/include/asm/reg.h index 562568414cf4..486b7c83b8c5 100644 --- a/arch/powerpc/include/asm/reg.h +++ b/arch/powerpc/include/asm/reg.h @@ -13,6 +13,8 @@ #include <linux/stringify.h> #include <asm/cputable.h> +#include <asm/asm-const.h> +#include <asm/feature-fixups.h> /* Pickup Book E specific registers. */ #if defined(CONFIG_BOOKE) || defined(CONFIG_40x) diff --git a/arch/powerpc/include/asm/reg_a2.h b/arch/powerpc/include/asm/reg_a2.h index 3ba9c6f096fc..74c2c57c492a 100644 --- a/arch/powerpc/include/asm/reg_a2.h +++ b/arch/powerpc/include/asm/reg_a2.h @@ -12,6 +12,8 @@ #ifndef __ASM_POWERPC_REG_A2_H__ #define __ASM_POWERPC_REG_A2_H__ +#include <asm/asm-const.h> + #define SPRN_TENSR 0x1b5 #define SPRN_TENS 0x1b6 /* Thread ENable Set */ #define SPRN_TENC 0x1b7 /* Thread ENable Clear */ diff --git a/arch/powerpc/include/asm/reg_fsl_emb.h b/arch/powerpc/include/asm/reg_fsl_emb.h index d7ccf93e6279..a21f529c43d9 100644 --- a/arch/powerpc/include/asm/reg_fsl_emb.h +++ b/arch/powerpc/include/asm/reg_fsl_emb.h @@ -7,6 +7,8 @@ #ifndef __ASM_POWERPC_REG_FSL_EMB_H__ #define __ASM_POWERPC_REG_FSL_EMB_H__ +#include <linux/stringify.h> + #ifndef __ASSEMBLY__ /* Performance Monitor Registers */ #define mfpmr(rn) ({unsigned int rval; \ diff --git a/arch/powerpc/include/asm/security_features.h b/arch/powerpc/include/asm/security_features.h index 44989b22383c..759597bf0fd8 100644 --- a/arch/powerpc/include/asm/security_features.h +++ b/arch/powerpc/include/asm/security_features.h @@ -22,6 +22,7 @@ enum stf_barrier_type { void setup_stf_barrier(void); void do_stf_barrier_fixups(enum stf_barrier_type types); +void setup_count_cache_flush(void); static inline void security_ftr_set(unsigned long feature) { @@ -59,6 +60,9 @@ static inline bool security_ftr_enabled(unsigned long feature) // Indirect branch prediction cache disabled #define SEC_FTR_COUNT_CACHE_DISABLED 0x0000000000000020ull +// bcctr 2,0,0 triggers a hardware assisted count cache flush +#define SEC_FTR_BCCTR_FLUSH_ASSIST 0x0000000000000800ull + // Features indicating need for Spectre/Meltdown mitigations @@ -74,6 +78,9 @@ static inline bool security_ftr_enabled(unsigned long feature) // Firmware configuration indicates user favours security over performance #define SEC_FTR_FAVOUR_SECURITY 0x0000000000000200ull +// Software required to flush count cache on context switch +#define SEC_FTR_FLUSH_COUNT_CACHE 0x0000000000000400ull + // Features enabled by default #define SEC_FTR_DEFAULT \ diff --git a/arch/powerpc/include/asm/setup.h b/arch/powerpc/include/asm/setup.h index 8721fd004291..1a951b00465d 100644 --- a/arch/powerpc/include/asm/setup.h +++ b/arch/powerpc/include/asm/setup.h @@ -52,11 +52,15 @@ enum l1d_flush_type { void setup_rfi_flush(enum l1d_flush_type, bool enable); void do_rfi_flush_fixups(enum l1d_flush_type types); +#ifdef CONFIG_PPC_BARRIER_NOSPEC void setup_barrier_nospec(void); +#else +static inline void setup_barrier_nospec(void) { }; +#endif void do_barrier_nospec_fixups(bool enable); extern bool barrier_nospec_enabled; -#ifdef CONFIG_PPC_BOOK3S_64 +#ifdef CONFIG_PPC_BARRIER_NOSPEC void do_barrier_nospec_fixups_range(bool enable, void *start, void *end); #else static inline void do_barrier_nospec_fixups_range(bool enable, void *start, void *end) { }; diff --git a/arch/powerpc/include/asm/smp.h b/arch/powerpc/include/asm/smp.h index 29ffaabdf75b..95b66a0c639b 100644 --- a/arch/powerpc/include/asm/smp.h +++ b/arch/powerpc/include/asm/smp.h @@ -56,7 +56,6 @@ struct smp_ops_t { int (*cpu_bootable)(unsigned int nr); }; -extern void smp_flush_nmi_ipi(u64 delay_us); extern int smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us); extern int smp_send_safe_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us); extern void smp_send_debugger_break(void); diff --git a/arch/powerpc/include/asm/sparsemem.h b/arch/powerpc/include/asm/sparsemem.h index bc66712bdc3c..28f5dae25db6 100644 --- a/arch/powerpc/include/asm/sparsemem.h +++ b/arch/powerpc/include/asm/sparsemem.h @@ -6,13 +6,20 @@ #ifdef CONFIG_SPARSEMEM /* * SECTION_SIZE_BITS 2^N: how big each section will be - * MAX_PHYSADDR_BITS 2^N: how much physical address space we have * MAX_PHYSMEM_BITS 2^N: how much memory we can have in that space */ #define SECTION_SIZE_BITS 24 - -#define MAX_PHYSADDR_BITS 46 +/* + * If we store section details in page->flags we can't increase the MAX_PHYSMEM_BITS + * if we increase SECTIONS_WIDTH we will not store node details in page->flags and + * page_to_nid does a page->section->node lookup + * Hence only increase for VMEMMAP. + */ +#ifdef CONFIG_SPARSEMEM_VMEMMAP +#define MAX_PHYSMEM_BITS 47 +#else #define MAX_PHYSMEM_BITS 46 +#endif #endif /* CONFIG_SPARSEMEM */ diff --git a/arch/powerpc/include/asm/spinlock.h b/arch/powerpc/include/asm/spinlock.h index 72dc4ddc2972..685c72310f5d 100644 --- a/arch/powerpc/include/asm/spinlock.h +++ b/arch/powerpc/include/asm/spinlock.h @@ -24,9 +24,9 @@ #include <asm/paca.h> #include <asm/hvcall.h> #endif -#include <asm/asm-compat.h> #include <asm/synch.h> #include <asm/ppc-opcode.h> +#include <asm/asm-405.h> #ifdef CONFIG_PPC64 /* use 0x800000yy when locked, where yy == CPU number */ diff --git a/arch/powerpc/include/asm/stacktrace.h b/arch/powerpc/include/asm/stacktrace.h new file mode 100644 index 000000000000..6149b53b3bc8 --- /dev/null +++ b/arch/powerpc/include/asm/stacktrace.h @@ -0,0 +1,13 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * Stack trace functions. + * + * Copyright 2018, Murilo Opsfelder Araujo, IBM Corporation. + */ + +#ifndef _ASM_POWERPC_STACKTRACE_H +#define _ASM_POWERPC_STACKTRACE_H + +void show_user_instructions(struct pt_regs *regs); + +#endif /* _ASM_POWERPC_STACKTRACE_H */ diff --git a/arch/powerpc/include/asm/string.h b/arch/powerpc/include/asm/string.h index 9b8cedf618f4..1647de15a31e 100644 --- a/arch/powerpc/include/asm/string.h +++ b/arch/powerpc/include/asm/string.h @@ -50,6 +50,8 @@ static inline void *memset64(uint64_t *p, uint64_t v, __kernel_size_t n) return __memset64(p, v, n * 8); } #else +#define __HAVE_ARCH_STRLEN + extern void *memset16(uint16_t *, uint16_t, __kernel_size_t); #endif #endif /* __KERNEL__ */ diff --git a/arch/powerpc/include/asm/synch.h b/arch/powerpc/include/asm/synch.h index 6ec546090ba1..aca70fb43147 100644 --- a/arch/powerpc/include/asm/synch.h +++ b/arch/powerpc/include/asm/synch.h @@ -3,8 +3,8 @@ #define _ASM_POWERPC_SYNCH_H #ifdef __KERNEL__ -#include <linux/stringify.h> #include <asm/feature-fixups.h> +#include <asm/asm-const.h> #ifndef __ASSEMBLY__ extern unsigned int __start___lwsync_fixup, __stop___lwsync_fixup; diff --git a/arch/powerpc/include/asm/thread_info.h b/arch/powerpc/include/asm/thread_info.h index f308dfeb2746..3c0002044bc9 100644 --- a/arch/powerpc/include/asm/thread_info.h +++ b/arch/powerpc/include/asm/thread_info.h @@ -9,6 +9,8 @@ #ifndef _ASM_POWERPC_THREAD_INFO_H #define _ASM_POWERPC_THREAD_INFO_H +#include <asm/asm-const.h> + #ifdef __KERNEL__ #define THREAD_SHIFT CONFIG_THREAD_SHIFT @@ -25,7 +27,6 @@ #include <linux/cache.h> #include <asm/processor.h> #include <asm/page.h> -#include <linux/stringify.h> #include <asm/accounting.h> /* diff --git a/arch/powerpc/include/asm/tlb.h b/arch/powerpc/include/asm/tlb.h index 9138baccebb0..f0e571b2dc7c 100644 --- a/arch/powerpc/include/asm/tlb.h +++ b/arch/powerpc/include/asm/tlb.h @@ -17,7 +17,6 @@ #include <asm/pgtable.h> #endif #include <asm/pgalloc.h> -#include <asm/tlbflush.h> #ifndef __powerpc64__ #include <asm/page.h> #include <asm/mmu.h> @@ -53,7 +52,8 @@ static inline void tlb_remove_check_page_size_change(struct mmu_gather *tlb, if (!tlb->page_size) tlb->page_size = page_size; else if (tlb->page_size != page_size) { - tlb_flush_mmu(tlb); + if (!tlb->fullmm) + tlb_flush_mmu(tlb); /* * update the page size after flush for the new * mmu_gather. diff --git a/arch/powerpc/include/asm/tlbflush.h b/arch/powerpc/include/asm/tlbflush.h index 7d5a157c7832..61fba43bf8b2 100644 --- a/arch/powerpc/include/asm/tlbflush.h +++ b/arch/powerpc/include/asm/tlbflush.h @@ -1,87 +1,11 @@ +/* SPDX-License-Identifier: GPL-2.0 */ #ifndef _ASM_POWERPC_TLBFLUSH_H #define _ASM_POWERPC_TLBFLUSH_H -/* - * TLB flushing: - * - * - flush_tlb_mm(mm) flushes the specified mm context TLB's - * - flush_tlb_page(vma, vmaddr) flushes one page - * - local_flush_tlb_mm(mm, full) flushes the specified mm context on - * the local processor - * - local_flush_tlb_page(vma, vmaddr) flushes one page on the local processor - * - flush_tlb_page_nohash(vma, vmaddr) flushes one page if SW loaded TLB - * - flush_tlb_range(vma, start, end) flushes a range of pages - * - flush_tlb_kernel_range(start, end) flushes a range of kernel pages - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ -#ifdef __KERNEL__ - -#ifdef CONFIG_PPC_MMU_NOHASH -/* - * TLB flushing for software loaded TLB chips - * - * TODO: (CONFIG_FSL_BOOKE) determine if flush_tlb_range & - * flush_tlb_kernel_range are best implemented as tlbia vs - * specific tlbie's - */ - -struct vm_area_struct; -struct mm_struct; - -#define MMU_NO_CONTEXT ((unsigned int)-1) - -extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end); -extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); - -extern void local_flush_tlb_mm(struct mm_struct *mm); -extern void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); - -extern void __local_flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, - int tsize, int ind); - -#ifdef CONFIG_SMP -extern void flush_tlb_mm(struct mm_struct *mm); -extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); -extern void __flush_tlb_page(struct mm_struct *mm, unsigned long vmaddr, - int tsize, int ind); -#else -#define flush_tlb_mm(mm) local_flush_tlb_mm(mm) -#define flush_tlb_page(vma,addr) local_flush_tlb_page(vma,addr) -#define __flush_tlb_page(mm,addr,p,i) __local_flush_tlb_page(mm,addr,p,i) -#endif - -#elif defined(CONFIG_PPC_STD_MMU_32) - -#define MMU_NO_CONTEXT (0) -/* - * TLB flushing for "classic" hash-MMU 32-bit CPUs, 6xx, 7xx, 7xxx - */ -extern void flush_tlb_mm(struct mm_struct *mm); -extern void flush_tlb_page(struct vm_area_struct *vma, unsigned long vmaddr); -extern void flush_tlb_page_nohash(struct vm_area_struct *vma, unsigned long addr); -extern void flush_tlb_range(struct vm_area_struct *vma, unsigned long start, - unsigned long end); -extern void flush_tlb_kernel_range(unsigned long start, unsigned long end); -static inline void local_flush_tlb_page(struct vm_area_struct *vma, - unsigned long vmaddr) -{ - flush_tlb_page(vma, vmaddr); -} -static inline void local_flush_tlb_mm(struct mm_struct *mm) -{ - flush_tlb_mm(mm); -} - -#elif defined(CONFIG_PPC_BOOK3S_64) -#include <asm/book3s/64/tlbflush.h> +#ifdef CONFIG_PPC_BOOK3S +#include <asm/book3s/tlbflush.h> #else -#error Unsupported MMU type -#endif +#include <asm/nohash/tlbflush.h> +#endif /* !CONFIG_PPC_BOOK3S */ -#endif /*__KERNEL__ */ #endif /* _ASM_POWERPC_TLBFLUSH_H */ diff --git a/arch/powerpc/include/asm/uaccess.h b/arch/powerpc/include/asm/uaccess.h index 468653ce844c..bac225bb7f64 100644 --- a/arch/powerpc/include/asm/uaccess.h +++ b/arch/powerpc/include/asm/uaccess.h @@ -2,7 +2,6 @@ #ifndef _ARCH_POWERPC_UACCESS_H #define _ARCH_POWERPC_UACCESS_H -#include <asm/asm-compat.h> #include <asm/ppc_asm.h> #include <asm/processor.h> #include <asm/page.h> @@ -250,10 +249,17 @@ do { \ } \ } while (0) +/* + * This is a type: either unsigned long, if the argument fits into + * that type, or otherwise unsigned long long. + */ +#define __long_type(x) \ + __typeof__(__builtin_choose_expr(sizeof(x) > sizeof(0UL), 0ULL, 0UL)) + #define __get_user_nocheck(x, ptr, size) \ ({ \ long __gu_err; \ - unsigned long __gu_val; \ + __long_type(*(ptr)) __gu_val; \ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ __chk_user_ptr(ptr); \ if (!is_kernel_addr((unsigned long)__gu_addr)) \ @@ -267,7 +273,7 @@ do { \ #define __get_user_check(x, ptr, size) \ ({ \ long __gu_err = -EFAULT; \ - unsigned long __gu_val = 0; \ + __long_type(*(ptr)) __gu_val = 0; \ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ might_fault(); \ if (access_ok(VERIFY_READ, __gu_addr, (size))) { \ @@ -281,7 +287,7 @@ do { \ #define __get_user_nosleep(x, ptr, size) \ ({ \ long __gu_err; \ - unsigned long __gu_val; \ + __long_type(*(ptr)) __gu_val; \ const __typeof__(*(ptr)) __user *__gu_addr = (ptr); \ __chk_user_ptr(ptr); \ barrier_nospec(); \ diff --git a/arch/powerpc/include/asm/xive.h b/arch/powerpc/include/asm/xive.h index 8d1a2792484f..3c704f5dd3ae 100644 --- a/arch/powerpc/include/asm/xive.h +++ b/arch/powerpc/include/asm/xive.h @@ -87,7 +87,6 @@ extern int xive_smp_prepare_cpu(unsigned int cpu); extern void xive_smp_setup_cpu(void); extern void xive_smp_disable_cpu(void); extern void xive_teardown_cpu(void); -extern void xive_kexec_teardown_cpu(int secondary); extern void xive_shutdown(void); extern void xive_flush_interrupt(void); diff --git a/arch/powerpc/kernel/Makefile b/arch/powerpc/kernel/Makefile index 2b4c40b255e4..3b66f2c19c84 100644 --- a/arch/powerpc/kernel/Makefile +++ b/arch/powerpc/kernel/Makefile @@ -7,10 +7,10 @@ CFLAGS_ptrace.o += -DUTS_MACHINE='"$(UTS_MACHINE)"' subdir-ccflags-$(CONFIG_PPC_WERROR) := -Werror -ifeq ($(CONFIG_PPC64),y) +ifdef CONFIG_PPC64 CFLAGS_prom_init.o += $(NO_MINIMAL_TOC) endif -ifeq ($(CONFIG_PPC32),y) +ifdef CONFIG_PPC32 CFLAGS_prom_init.o += -fPIC CFLAGS_btext.o += -fPIC endif @@ -42,9 +42,10 @@ obj-$(CONFIG_VDSO32) += vdso32/ obj-$(CONFIG_PPC_WATCHDOG) += watchdog.o obj-$(CONFIG_HAVE_HW_BREAKPOINT) += hw_breakpoint.o obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_ppc970.o cpu_setup_pa6t.o -obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_power.o security.o +obj-$(CONFIG_PPC_BOOK3S_64) += cpu_setup_power.o obj-$(CONFIG_PPC_BOOK3S_64) += mce.o mce_power.o obj-$(CONFIG_PPC_BOOK3E_64) += exceptions-64e.o idle_book3e.o +obj-$(CONFIG_PPC_BARRIER_NOSPEC) += security.o obj-$(CONFIG_PPC64) += vdso64/ obj-$(CONFIG_ALTIVEC) += vecemu.o obj-$(CONFIG_PPC_970_NAP) += idle_power4.o @@ -62,13 +63,13 @@ obj-$(CONFIG_EEH) += eeh.o eeh_pe.o eeh_dev.o eeh_cache.o \ obj-$(CONFIG_GENERIC_TBSYNC) += smp-tbsync.o obj-$(CONFIG_CRASH_DUMP) += crash_dump.o obj-$(CONFIG_FA_DUMP) += fadump.o -ifeq ($(CONFIG_PPC32),y) +ifdef CONFIG_PPC32 obj-$(CONFIG_E500) += idle_e500.o endif obj-$(CONFIG_6xx) += idle_6xx.o l2cr_6xx.o cpu_setup_6xx.o obj-$(CONFIG_TAU) += tau_6xx.o obj-$(CONFIG_HIBERNATION) += swsusp.o suspend.o -ifeq ($(CONFIG_FSL_BOOKE),y) +ifdef CONFIG_FSL_BOOKE obj-$(CONFIG_HIBERNATION) += swsusp_booke.o else obj-$(CONFIG_HIBERNATION) += swsusp_$(BITS).o @@ -109,9 +110,11 @@ obj-$(CONFIG_PCI_MSI) += msi.o obj-$(CONFIG_KEXEC_CORE) += machine_kexec.o crash.o \ machine_kexec_$(BITS).o obj-$(CONFIG_KEXEC_FILE) += machine_kexec_file_$(BITS).o kexec_elf_$(BITS).o -ifeq ($(CONFIG_HAVE_IMA_KEXEC)$(CONFIG_IMA),yy) +ifdef CONFIG_HAVE_IMA_KEXEC +ifdef CONFIG_IMA obj-y += ima_kexec.o endif +endif obj-$(CONFIG_AUDIT) += audit.o obj64-$(CONFIG_AUDIT) += compat_audit.o @@ -164,7 +167,7 @@ PHONY += systbl_chk systbl_chk: $(src)/systbl_chk.sh $(obj)/systbl_chk.i $(call cmd,systbl_chk) -ifeq ($(CONFIG_PPC_OF_BOOT_TRAMPOLINE),y) +ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE $(obj)/built-in.a: prom_init_check quiet_cmd_prom_init_check = CALL $< diff --git a/arch/powerpc/kernel/asm-offsets.c b/arch/powerpc/kernel/asm-offsets.c index 0a0544335950..89cf15566c4e 100644 --- a/arch/powerpc/kernel/asm-offsets.c +++ b/arch/powerpc/kernel/asm-offsets.c @@ -766,7 +766,6 @@ int main(void) OFFSET(PACA_THREAD_IDLE_STATE, paca_struct, thread_idle_state); OFFSET(PACA_THREAD_MASK, paca_struct, thread_mask); OFFSET(PACA_SUBCORE_SIBLING_MASK, paca_struct, subcore_sibling_mask); - OFFSET(PACA_SIBLING_PACA_PTRS, paca_struct, thread_sibling_pacas); OFFSET(PACA_REQ_PSSCR, paca_struct, requested_psscr); OFFSET(PACA_DONT_STOP, paca_struct, dont_stop); #define STOP_SPR(x, f) OFFSET(x, paca_struct, stop_sprs.f) diff --git a/arch/powerpc/kernel/cpu_setup_6xx.S b/arch/powerpc/kernel/cpu_setup_6xx.S index a9f3970693e1..fa3c2c91290c 100644 --- a/arch/powerpc/kernel/cpu_setup_6xx.S +++ b/arch/powerpc/kernel/cpu_setup_6xx.S @@ -16,6 +16,7 @@ #include <asm/asm-offsets.h> #include <asm/cache.h> #include <asm/mmu.h> +#include <asm/feature-fixups.h> _GLOBAL(__setup_cpu_603) mflr r5 diff --git a/arch/powerpc/kernel/cputable.c b/arch/powerpc/kernel/cputable.c index c8fc9691f8c7..2da01340c84c 100644 --- a/arch/powerpc/kernel/cputable.c +++ b/arch/powerpc/kernel/cputable.c @@ -447,25 +447,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", }, - { /* Power8 DD1: Does not support doorbell IPIs */ - .pvr_mask = 0xffffff00, - .pvr_value = 0x004d0100, - .cpu_name = "POWER8 (raw)", - .cpu_features = CPU_FTRS_POWER8_DD1, - .cpu_user_features = COMMON_USER_POWER8, - .cpu_user_features2 = COMMON_USER2_POWER8, - .mmu_features = MMU_FTRS_POWER8, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .oprofile_cpu_type = "ppc64/power8", - .oprofile_type = PPC_OPROFILE_INVALID, - .cpu_setup = __setup_cpu_power8, - .cpu_restore = __restore_cpu_power8, - .machine_check_early = __machine_check_early_realmode_p8, - .platform = "power8", - }, { /* Power8 */ .pvr_mask = 0xffff0000, .pvr_value = 0x004d0000, @@ -485,25 +466,6 @@ static struct cpu_spec __initdata cpu_specs[] = { .machine_check_early = __machine_check_early_realmode_p8, .platform = "power8", }, - { /* Power9 DD1*/ - .pvr_mask = 0xffffff00, - .pvr_value = 0x004e0100, - .cpu_name = "POWER9 (raw)", - .cpu_features = CPU_FTRS_POWER9_DD1, - .cpu_user_features = COMMON_USER_POWER9, - .cpu_user_features2 = COMMON_USER2_POWER9, - .mmu_features = MMU_FTRS_POWER9, - .icache_bsize = 128, - .dcache_bsize = 128, - .num_pmcs = 6, - .pmc_type = PPC_PMC_IBM, - .oprofile_cpu_type = "ppc64/power9", - .oprofile_type = PPC_OPROFILE_INVALID, - .cpu_setup = __setup_cpu_power9, - .cpu_restore = __restore_cpu_power9, - .machine_check_early = __machine_check_early_realmode_p9, - .platform = "power9", - }, { /* Power9 DD2.0 */ .pvr_mask = 0xffffefff, .pvr_value = 0x004e0200, diff --git a/arch/powerpc/kernel/crash.c b/arch/powerpc/kernel/crash.c index 17c8b99680f2..43a3ce2301e8 100644 --- a/arch/powerpc/kernel/crash.c +++ b/arch/powerpc/kernel/crash.c @@ -23,7 +23,6 @@ #include <asm/processor.h> #include <asm/machdep.h> #include <asm/kexec.h> -#include <asm/kdump.h> #include <asm/prom.h> #include <asm/smp.h> #include <asm/setjmp.h> diff --git a/arch/powerpc/kernel/dt_cpu_ftrs.c b/arch/powerpc/kernel/dt_cpu_ftrs.c index 96dd3d871986..f432054234a4 100644 --- a/arch/powerpc/kernel/dt_cpu_ftrs.c +++ b/arch/powerpc/kernel/dt_cpu_ftrs.c @@ -701,9 +701,7 @@ static __init void cpufeatures_cpu_quirks(void) /* * Not all quirks can be derived from the cpufeatures device tree. */ - if ((version & 0xffffff00) == 0x004e0100) - cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD1; - else if ((version & 0xffffefff) == 0x004e0200) + if ((version & 0xffffefff) == 0x004e0200) ; /* DD2.0 has no feature flag */ else if ((version & 0xffffefff) == 0x004e0201) cur_cpu_spec->cpu_features |= CPU_FTR_POWER9_DD2_1; diff --git a/arch/powerpc/kernel/eeh.c b/arch/powerpc/kernel/eeh.c index 5746809cfaad..6ebba3e48b01 100644 --- a/arch/powerpc/kernel/eeh.c +++ b/arch/powerpc/kernel/eeh.c @@ -1087,7 +1087,7 @@ static int eeh_init(void) if (eeh_enabled()) pr_info("EEH: PCI Enhanced I/O Error Handling Enabled\n"); - else + else if (!eeh_has_flag(EEH_POSTPONED_PROBE)) pr_info("EEH: No capable adapters found\n"); return ret; diff --git a/arch/powerpc/kernel/entry_32.S b/arch/powerpc/kernel/entry_32.S index 973577f2141c..e58c3f467db5 100644 --- a/arch/powerpc/kernel/entry_32.S +++ b/arch/powerpc/kernel/entry_32.S @@ -33,6 +33,9 @@ #include <asm/unistd.h> #include <asm/ptrace.h> #include <asm/export.h> +#include <asm/asm-405.h> +#include <asm/feature-fixups.h> +#include <asm/barrier.h> /* * MSR_KERNEL is > 0x10000 on 4xx/Book-E since it include MSR_CE. @@ -358,6 +361,15 @@ syscall_dotrace_cont: ori r10,r10,sys_call_table@l slwi r0,r0,2 bge- 66f + + barrier_nospec_asm + /* + * Prevent the load of the handler below (based on the user-passed + * system call number) being speculatively executed until the test + * against NR_syscalls and branch to .66f above has + * committed. + */ + lwzx r10,r10,r0 /* Fetch system call handler [ptr] */ mtlr r10 addi r9,r1,STACK_FRAME_OVERHEAD diff --git a/arch/powerpc/kernel/entry_64.S b/arch/powerpc/kernel/entry_64.S index 729e9ef4d3bb..2206912ea4f0 100644 --- a/arch/powerpc/kernel/entry_64.S +++ b/arch/powerpc/kernel/entry_64.S @@ -25,6 +25,7 @@ #include <asm/page.h> #include <asm/mmu.h> #include <asm/thread_info.h> +#include <asm/code-patching-asm.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/cputable.h> @@ -38,11 +39,13 @@ #include <asm/ppc-opcode.h> #include <asm/barrier.h> #include <asm/export.h> +#include <asm/asm-compat.h> #ifdef CONFIG_PPC_BOOK3S #include <asm/exception-64s.h> #else #include <asm/exception-64e.h> #endif +#include <asm/feature-fixups.h> /* * System calls. @@ -504,6 +507,57 @@ _GLOBAL(ret_from_kernel_thread) li r3,0 b .Lsyscall_exit +#ifdef CONFIG_PPC_BOOK3S_64 + +#define FLUSH_COUNT_CACHE \ +1: nop; \ + patch_site 1b, patch__call_flush_count_cache + + +#define BCCTR_FLUSH .long 0x4c400420 + +.macro nops number + .rept \number + nop + .endr +.endm + +.balign 32 +.global flush_count_cache +flush_count_cache: + /* Save LR into r9 */ + mflr r9 + + .rept 64 + bl .+4 + .endr + b 1f + nops 6 + + .balign 32 + /* Restore LR */ +1: mtlr r9 + li r9,0x7fff + mtctr r9 + + BCCTR_FLUSH + +2: nop + patch_site 2b patch__flush_count_cache_return + + nops 3 + + .rept 278 + .balign 32 + BCCTR_FLUSH + nops 7 + .endr + + blr +#else +#define FLUSH_COUNT_CACHE +#endif /* CONFIG_PPC_BOOK3S_64 */ + /* * This routine switches between two different tasks. The process * state of one is saved on its kernel stack. Then the state @@ -535,6 +589,8 @@ _GLOBAL(_switch) std r23,_CCR(r1) std r1,KSP(r3) /* Set old stack pointer */ + FLUSH_COUNT_CACHE + /* * On SMP kernels, care must be taken because a task may be * scheduled off CPUx and on to CPUy. Memory ordering must be @@ -993,6 +1049,14 @@ END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR) std r4,_TRAP(r1) /* + * PACA_IRQ_HARD_DIS won't always be set here, so set it now + * to reconcile the IRQ state. Tracing is already accounted for. + */ + lbz r4,PACAIRQHAPPENED(r13) + ori r4,r4,PACA_IRQ_HARD_DIS + stb r4,PACAIRQHAPPENED(r13) + + /* * Then find the right handler and call it. Interrupts are * still soft-disabled and we keep them that way. */ diff --git a/arch/powerpc/kernel/exceptions-64e.S b/arch/powerpc/kernel/exceptions-64e.S index 9b6e653e501a..6d6e144a28ce 100644 --- a/arch/powerpc/kernel/exceptions-64e.S +++ b/arch/powerpc/kernel/exceptions-64e.S @@ -27,6 +27,7 @@ #include <asm/hw_irq.h> #include <asm/kvm_asm.h> #include <asm/kvm_booke_hv_asm.h> +#include <asm/feature-fixups.h> /* XXX This will ultimately add space for a special exception save * structure used to save things like SRR0/SRR1, SPRGs, MAS, etc... @@ -949,7 +950,11 @@ kernel_dbg_exc: .macro masked_interrupt_book3e paca_irq full_mask lbz r10,PACAIRQHAPPENED(r13) + .if \full_mask == 1 + ori r10,r10,\paca_irq | PACA_IRQ_HARD_DIS + .else ori r10,r10,\paca_irq + .endif stb r10,PACAIRQHAPPENED(r13) .if \full_mask == 1 diff --git a/arch/powerpc/kernel/exceptions-64s.S b/arch/powerpc/kernel/exceptions-64s.S index 285c6465324a..ea04dfb8c092 100644 --- a/arch/powerpc/kernel/exceptions-64s.S +++ b/arch/powerpc/kernel/exceptions-64s.S @@ -18,6 +18,7 @@ #include <asm/ptrace.h> #include <asm/cpuidle.h> #include <asm/head-64.h> +#include <asm/feature-fixups.h> /* * There are a few constraints to be concerned with. @@ -126,8 +127,8 @@ EXC_REAL_BEGIN(system_reset, 0x100, 0x100) * MSR_RI is not enabled, because PACA_EXNMI and nmi stack is * being used, so a nested NMI exception would corrupt it. */ - EXCEPTION_PROLOG_PSERIES_NORI(PACA_EXNMI, system_reset_common, EXC_STD, - IDLETEST, 0x100) + EXCEPTION_PROLOG_NORI(PACA_EXNMI, system_reset_common, EXC_STD, + IDLETEST, 0x100) EXC_REAL_END(system_reset, 0x100, 0x100) EXC_VIRT_NONE(0x4100, 0x100) @@ -230,8 +231,8 @@ EXC_COMMON_BEGIN(system_reset_common) TRAMP_REAL_BEGIN(system_reset_fwnmi) SET_SCRATCH0(r13) /* save r13 */ /* See comment at system_reset exception */ - EXCEPTION_PROLOG_PSERIES_NORI(PACA_EXNMI, system_reset_common, - EXC_STD, NOTEST, 0x100) + EXCEPTION_PROLOG_NORI(PACA_EXNMI, system_reset_common, EXC_STD, + NOTEST, 0x100) #endif /* CONFIG_PPC_PSERIES */ @@ -276,9 +277,7 @@ BEGIN_FTR_SECTION * * This interrupt can wake directly from idle. If that is the case, * the machine check is handled then the idle wakeup code is called - * to restore state. In that case, the POWER9 DD1 idle PACA workaround - * is not applied in the early machine check code, which will cause - * bugs. + * to restore state. */ mr r11,r1 /* Save r1 */ lhz r10,PACA_IN_MCE(r13) @@ -339,7 +338,7 @@ machine_check_pSeries_0: * nested machine check corrupts it. machine_check_common enables * MSR_RI. */ - EXCEPTION_PROLOG_PSERIES_1_NORI(machine_check_common, EXC_STD) + EXCEPTION_PROLOG_2_NORI(machine_check_common, EXC_STD) TRAMP_KVM_SKIP(PACA_EXMC, 0x200) @@ -769,13 +768,9 @@ EXC_REAL_BEGIN(hardware_interrupt, 0x500, 0x100) .globl hardware_interrupt_hv; hardware_interrupt_hv: BEGIN_FTR_SECTION - _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, - EXC_HV, SOFTEN_TEST_HV, - IRQS_DISABLED) + MASKABLE_EXCEPTION_HV(0x500, hardware_interrupt_common, IRQS_DISABLED) FTR_SECTION_ELSE - _MASKABLE_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, - EXC_STD, SOFTEN_TEST_PR, - IRQS_DISABLED) + MASKABLE_EXCEPTION(0x500, hardware_interrupt_common, IRQS_DISABLED) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE | CPU_FTR_ARCH_206) EXC_REAL_END(hardware_interrupt, 0x500, 0x100) @@ -783,13 +778,11 @@ EXC_VIRT_BEGIN(hardware_interrupt, 0x4500, 0x100) .globl hardware_interrupt_relon_hv; hardware_interrupt_relon_hv: BEGIN_FTR_SECTION - _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, - EXC_HV, SOFTEN_TEST_HV, - IRQS_DISABLED) + MASKABLE_RELON_EXCEPTION_HV(0x500, hardware_interrupt_common, + IRQS_DISABLED) FTR_SECTION_ELSE - _MASKABLE_RELON_EXCEPTION_PSERIES(0x500, hardware_interrupt_common, - EXC_STD, SOFTEN_TEST_PR, - IRQS_DISABLED) + __MASKABLE_RELON_EXCEPTION(0x500, hardware_interrupt_common, + EXC_STD, SOFTEN_TEST_PR, IRQS_DISABLED) ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE) EXC_VIRT_END(hardware_interrupt, 0x4500, 0x100) @@ -1328,7 +1321,7 @@ EXC_REAL_BEGIN(denorm_exception_hv, 0x1500, 0x100) #endif KVMTEST_HV(0x1500) - EXCEPTION_PROLOG_PSERIES_1(denorm_common, EXC_HV) + EXCEPTION_PROLOG_2(denorm_common, EXC_HV) EXC_REAL_END(denorm_exception_hv, 0x1500, 0x100) #ifdef CONFIG_PPC_DENORMALISATION @@ -1448,7 +1441,7 @@ EXC_VIRT_NONE(0x5800, 0x100) std r12,PACA_EXGEN+EX_R12(r13); \ GET_SCRATCH0(r10); \ std r10,PACA_EXGEN+EX_R13(r13); \ - EXCEPTION_PROLOG_PSERIES_1(soft_nmi_common, _H) + EXCEPTION_PROLOG_2(soft_nmi_common, _H) /* * Branch to soft_nmi_interrupt using the emergency stack. The emergency @@ -1500,7 +1493,10 @@ masked_##_H##interrupt: \ mfspr r10,SPRN_##_H##SRR1; \ xori r10,r10,MSR_EE; /* clear MSR_EE */ \ mtspr SPRN_##_H##SRR1,r10; \ -2: mtcrf 0x80,r9; \ + ori r11,r11,PACA_IRQ_HARD_DIS; \ + stb r11,PACAIRQHAPPENED(r13); \ +2: /* done */ \ + mtcrf 0x80,r9; \ std r1,PACAR1(r13); \ ld r9,PACA_EXGEN+EX_R9(r13); \ ld r10,PACA_EXGEN+EX_R10(r13); \ @@ -1526,6 +1522,8 @@ TRAMP_REAL_BEGIN(stf_barrier_fallback) TRAMP_REAL_BEGIN(rfi_flush_fallback) SET_SCRATCH0(r13); GET_PACA(r13); + std r1,PACA_EXRFI+EX_R12(r13) + ld r1,PACAKSAVE(r13) std r9,PACA_EXRFI+EX_R9(r13) std r10,PACA_EXRFI+EX_R10(r13) std r11,PACA_EXRFI+EX_R11(r13) @@ -1560,12 +1558,15 @@ TRAMP_REAL_BEGIN(rfi_flush_fallback) ld r9,PACA_EXRFI+EX_R9(r13) ld r10,PACA_EXRFI+EX_R10(r13) ld r11,PACA_EXRFI+EX_R11(r13) + ld r1,PACA_EXRFI+EX_R12(r13) GET_SCRATCH0(r13); rfid TRAMP_REAL_BEGIN(hrfi_flush_fallback) SET_SCRATCH0(r13); GET_PACA(r13); + std r1,PACA_EXRFI+EX_R12(r13) + ld r1,PACAKSAVE(r13) std r9,PACA_EXRFI+EX_R9(r13) std r10,PACA_EXRFI+EX_R10(r13) std r11,PACA_EXRFI+EX_R11(r13) @@ -1600,6 +1601,7 @@ TRAMP_REAL_BEGIN(hrfi_flush_fallback) ld r9,PACA_EXRFI+EX_R9(r13) ld r10,PACA_EXRFI+EX_R10(r13) ld r11,PACA_EXRFI+EX_R11(r13) + ld r1,PACA_EXRFI+EX_R12(r13) GET_SCRATCH0(r13); hrfid diff --git a/arch/powerpc/kernel/fadump.c b/arch/powerpc/kernel/fadump.c index 07e8396d472b..986ec476fd5d 100644 --- a/arch/powerpc/kernel/fadump.c +++ b/arch/powerpc/kernel/fadump.c @@ -47,8 +47,10 @@ static struct fadump_mem_struct fdm; static const struct fadump_mem_struct *fdm_active; static DEFINE_MUTEX(fadump_mutex); -struct fad_crash_memory_ranges crash_memory_ranges[INIT_CRASHMEM_RANGES]; +struct fad_crash_memory_ranges *crash_memory_ranges; +int crash_memory_ranges_size; int crash_mem_ranges; +int max_crash_mem_ranges; /* Scan the Firmware Assisted dump configuration details. */ int __init early_init_dt_scan_fw_dump(unsigned long node, @@ -868,38 +870,107 @@ static int __init process_fadump(const struct fadump_mem_struct *fdm_active) return 0; } -static inline void fadump_add_crash_memory(unsigned long long base, - unsigned long long end) +static void free_crash_memory_ranges(void) +{ + kfree(crash_memory_ranges); + crash_memory_ranges = NULL; + crash_memory_ranges_size = 0; + max_crash_mem_ranges = 0; +} + +/* + * Allocate or reallocate crash memory ranges array in incremental units + * of PAGE_SIZE. + */ +static int allocate_crash_memory_ranges(void) { + struct fad_crash_memory_ranges *new_array; + u64 new_size; + + new_size = crash_memory_ranges_size + PAGE_SIZE; + pr_debug("Allocating %llu bytes of memory for crash memory ranges\n", + new_size); + + new_array = krealloc(crash_memory_ranges, new_size, GFP_KERNEL); + if (new_array == NULL) { + pr_err("Insufficient memory for setting up crash memory ranges\n"); + free_crash_memory_ranges(); + return -ENOMEM; + } + + crash_memory_ranges = new_array; + crash_memory_ranges_size = new_size; + max_crash_mem_ranges = (new_size / + sizeof(struct fad_crash_memory_ranges)); + return 0; +} + +static inline int fadump_add_crash_memory(unsigned long long base, + unsigned long long end) +{ + u64 start, size; + bool is_adjacent = false; + if (base == end) - return; + return 0; + + /* + * Fold adjacent memory ranges to bring down the memory ranges/ + * PT_LOAD segments count. + */ + if (crash_mem_ranges) { + start = crash_memory_ranges[crash_mem_ranges - 1].base; + size = crash_memory_ranges[crash_mem_ranges - 1].size; + + if ((start + size) == base) + is_adjacent = true; + } + if (!is_adjacent) { + /* resize the array on reaching the limit */ + if (crash_mem_ranges == max_crash_mem_ranges) { + int ret; + + ret = allocate_crash_memory_ranges(); + if (ret) + return ret; + } + start = base; + crash_memory_ranges[crash_mem_ranges].base = start; + crash_mem_ranges++; + } + + crash_memory_ranges[crash_mem_ranges - 1].size = (end - start); pr_debug("crash_memory_range[%d] [%#016llx-%#016llx], %#llx bytes\n", - crash_mem_ranges, base, end - 1, (end - base)); - crash_memory_ranges[crash_mem_ranges].base = base; - crash_memory_ranges[crash_mem_ranges].size = end - base; - crash_mem_ranges++; + (crash_mem_ranges - 1), start, end - 1, (end - start)); + return 0; } -static void fadump_exclude_reserved_area(unsigned long long start, +static int fadump_exclude_reserved_area(unsigned long long start, unsigned long long end) { unsigned long long ra_start, ra_end; + int ret = 0; ra_start = fw_dump.reserve_dump_area_start; ra_end = ra_start + fw_dump.reserve_dump_area_size; if ((ra_start < end) && (ra_end > start)) { if ((start < ra_start) && (end > ra_end)) { - fadump_add_crash_memory(start, ra_start); - fadump_add_crash_memory(ra_end, end); + ret = fadump_add_crash_memory(start, ra_start); + if (ret) + return ret; + + ret = fadump_add_crash_memory(ra_end, end); } else if (start < ra_start) { - fadump_add_crash_memory(start, ra_start); + ret = fadump_add_crash_memory(start, ra_start); } else if (ra_end < end) { - fadump_add_crash_memory(ra_end, end); + ret = fadump_add_crash_memory(ra_end, end); } } else - fadump_add_crash_memory(start, end); + ret = fadump_add_crash_memory(start, end); + + return ret; } static int fadump_init_elfcore_header(char *bufp) @@ -939,13 +1010,22 @@ static int fadump_init_elfcore_header(char *bufp) * Traverse through memblock structure and setup crash memory ranges. These * ranges will be used create PT_LOAD program headers in elfcore header. */ -static void fadump_setup_crash_memory_ranges(void) +static int fadump_setup_crash_memory_ranges(void) { struct memblock_region *reg; unsigned long long start, end; + int ret; pr_debug("Setup crash memory ranges.\n"); crash_mem_ranges = 0; + + /* allocate memory for crash memory ranges for the first time */ + if (!max_crash_mem_ranges) { + ret = allocate_crash_memory_ranges(); + if (ret) + return ret; + } + /* * add the first memory chunk (RMA_START through boot_memory_size) as * a separate memory chunk. The reason is, at the time crash firmware @@ -953,7 +1033,9 @@ static void fadump_setup_crash_memory_ranges(void) * specified during fadump registration. We need to create a separate * program header for this chunk with the correct offset. */ - fadump_add_crash_memory(RMA_START, fw_dump.boot_memory_size); + ret = fadump_add_crash_memory(RMA_START, fw_dump.boot_memory_size); + if (ret) + return ret; for_each_memblock(memory, reg) { start = (unsigned long long)reg->base; @@ -973,8 +1055,12 @@ static void fadump_setup_crash_memory_ranges(void) } /* add this range excluding the reserved dump area. */ - fadump_exclude_reserved_area(start, end); + ret = fadump_exclude_reserved_area(start, end); + if (ret) + return ret; } + + return 0; } /* @@ -1097,6 +1183,7 @@ static int register_fadump(void) { unsigned long addr; void *vaddr; + int ret; /* * If no memory is reserved then we can not register for firmware- @@ -1105,7 +1192,9 @@ static int register_fadump(void) if (!fw_dump.reserve_dump_area_size) return -ENODEV; - fadump_setup_crash_memory_ranges(); + ret = fadump_setup_crash_memory_ranges(); + if (ret) + return ret; addr = be64_to_cpu(fdm.rmr_region.destination_address) + be64_to_cpu(fdm.rmr_region.source_len); /* Initialize fadump crash info header. */ @@ -1183,6 +1272,7 @@ void fadump_cleanup(void) } else if (fw_dump.dump_registered) { /* Un-register Firmware-assisted dump if it was registered. */ fadump_unregister_dump(&fdm); + free_crash_memory_ranges(); } } diff --git a/arch/powerpc/kernel/fpu.S b/arch/powerpc/kernel/fpu.S index 6c509f39bbde..529dcc21c3f9 100644 --- a/arch/powerpc/kernel/fpu.S +++ b/arch/powerpc/kernel/fpu.S @@ -25,6 +25,8 @@ #include <asm/asm-offsets.h> #include <asm/ptrace.h> #include <asm/export.h> +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> #ifdef CONFIG_VSX #define __REST_32FPVSRS(n,c,base) \ diff --git a/arch/powerpc/kernel/head_32.S b/arch/powerpc/kernel/head_32.S index 29b2fed93289..61ca27929355 100644 --- a/arch/powerpc/kernel/head_32.S +++ b/arch/powerpc/kernel/head_32.S @@ -35,6 +35,7 @@ #include <asm/bug.h> #include <asm/kvm_book3s_asm.h> #include <asm/export.h> +#include <asm/feature-fixups.h> /* 601 only have IBAT; cr0.eq is set on 601 when using this macro */ #define LOAD_BAT(n, reg, RA, RB) \ diff --git a/arch/powerpc/kernel/head_40x.S b/arch/powerpc/kernel/head_40x.S index 41374a468d1c..b19d78410511 100644 --- a/arch/powerpc/kernel/head_40x.S +++ b/arch/powerpc/kernel/head_40x.S @@ -42,6 +42,7 @@ #include <asm/asm-offsets.h> #include <asm/ptrace.h> #include <asm/export.h> +#include <asm/asm-405.h> /* As with the other PowerPC ports, it is expected that when code * execution begins here, the following registers contain valid, yet diff --git a/arch/powerpc/kernel/head_64.S b/arch/powerpc/kernel/head_64.S index 6eca15f25c73..4898e9491a1c 100644 --- a/arch/powerpc/kernel/head_64.S +++ b/arch/powerpc/kernel/head_64.S @@ -44,6 +44,7 @@ #include <asm/cputhreads.h> #include <asm/ppc-opcode.h> #include <asm/export.h> +#include <asm/feature-fixups.h> /* The physical memory is laid out such that the secondary processor * spin code sits at 0x0000...0x00ff. On server, the vectors follow diff --git a/arch/powerpc/kernel/head_8xx.S b/arch/powerpc/kernel/head_8xx.S index 6cab07e76732..6582f824d620 100644 --- a/arch/powerpc/kernel/head_8xx.S +++ b/arch/powerpc/kernel/head_8xx.S @@ -30,7 +30,6 @@ #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/ptrace.h> -#include <asm/fixmap.h> #include <asm/export.h> #if CONFIG_TASK_SIZE <= 0x80000000 && CONFIG_PAGE_OFFSET >= 0x80000000 @@ -873,6 +872,10 @@ start_here: li r0,0 stwu r0,THREAD_SIZE-STACK_FRAME_OVERHEAD(r1) + lis r6, swapper_pg_dir@ha + tophys(r6,r6) + mtspr SPRN_M_TW, r6 + bl early_init /* We have to do this with MMU on */ /* @@ -893,9 +896,6 @@ start_here: * init's THREAD like the context switch code does, but this is * easier......until someone changes init's static structures. */ - lis r6, swapper_pg_dir@ha - tophys(r6,r6) - mtspr SPRN_M_TW, r6 lis r4,2f@h ori r4,r4,2f@l tophys(r4,r4) diff --git a/arch/powerpc/kernel/head_fsl_booke.S b/arch/powerpc/kernel/head_fsl_booke.S index bf4c6021515f..e2750b856c8f 100644 --- a/arch/powerpc/kernel/head_fsl_booke.S +++ b/arch/powerpc/kernel/head_fsl_booke.S @@ -43,6 +43,7 @@ #include <asm/cache.h> #include <asm/ptrace.h> #include <asm/export.h> +#include <asm/feature-fixups.h> #include "head_booke.h" /* As with the other PowerPC ports, it is expected that when code diff --git a/arch/powerpc/kernel/idle_6xx.S b/arch/powerpc/kernel/idle_6xx.S index 1686916cc7f0..ff026c9d3cab 100644 --- a/arch/powerpc/kernel/idle_6xx.S +++ b/arch/powerpc/kernel/idle_6xx.S @@ -20,6 +20,7 @@ #include <asm/thread_info.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/feature-fixups.h> .text diff --git a/arch/powerpc/kernel/idle_book3e.S b/arch/powerpc/kernel/idle_book3e.S index 2b269315d377..4e0d94d02030 100644 --- a/arch/powerpc/kernel/idle_book3e.S +++ b/arch/powerpc/kernel/idle_book3e.S @@ -36,7 +36,7 @@ _GLOBAL(\name) */ lbz r3,PACAIRQHAPPENED(r13) cmpwi cr0,r3,0 - bnelr + bne 2f /* Now we are going to mark ourselves as soft and hard enabled in * order to be able to take interrupts while asleep. We inform lockdep @@ -72,6 +72,11 @@ _GLOBAL(\name) wrteei 1 \loop +2: + lbz r10,PACAIRQHAPPENED(r13) + ori r10,r10,PACA_IRQ_HARD_DIS + stb r10,PACAIRQHAPPENED(r13) + blr .endm .macro BOOK3E_IDLE_LOOP diff --git a/arch/powerpc/kernel/idle_book3s.S b/arch/powerpc/kernel/idle_book3s.S index 689306118b48..7f5ac2e8581b 100644 --- a/arch/powerpc/kernel/idle_book3s.S +++ b/arch/powerpc/kernel/idle_book3s.S @@ -23,6 +23,8 @@ #include <asm/exception-64s.h> #include <asm/book3s/64/mmu-hash.h> #include <asm/mmu.h> +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> #undef DEBUG @@ -469,43 +471,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_P9_TM_XER_SO_BUG) #endif /* - * On waking up from stop 0,1,2 with ESL=1 on POWER9 DD1, - * HSPRG0 will be set to the HSPRG0 value of one of the - * threads in this core. Thus the value we have in r13 - * may not be this thread's paca pointer. - * - * Fortunately, the TIR remains invariant. Since this thread's - * paca pointer is recorded in all its sibling's paca, we can - * correctly recover this thread's paca pointer if we - * know the index of this thread in the core. - * - * This index can be obtained from the TIR. - * - * i.e, thread's position in the core = TIR. - * If this value is i, then this thread's paca is - * paca->thread_sibling_pacas[i]. - */ -power9_dd1_recover_paca: - mfspr r4, SPRN_TIR - /* - * Since each entry in thread_sibling_pacas is 8 bytes - * we need to left-shift by 3 bits. Thus r4 = i * 8 - */ - sldi r4, r4, 3 - /* Get &paca->thread_sibling_pacas[0] in r5 */ - ld r5, PACA_SIBLING_PACA_PTRS(r13) - /* Load paca->thread_sibling_pacas[i] into r13 */ - ldx r13, r4, r5 - SET_PACA(r13) - /* - * Indicate that we have lost NVGPR state - * which needs to be restored from the stack. - */ - li r3, 1 - stb r3,PACA_NAPSTATELOST(r13) - blr - -/* * Called from machine check handler for powersave wakeups. * Low level machine check processing has already been done. Now just * go through the wake up path to get everything in order. @@ -539,9 +504,6 @@ pnv_powersave_wakeup: ld r2, PACATOC(r13) BEGIN_FTR_SECTION -BEGIN_FTR_SECTION_NESTED(70) - bl power9_dd1_recover_paca -END_FTR_SECTION_NESTED_IFSET(CPU_FTR_POWER9_DD1, 70) bl pnv_restore_hyp_resource_arch300 FTR_SECTION_ELSE bl pnv_restore_hyp_resource_arch207 @@ -604,22 +566,12 @@ END_FTR_SECTION_IFCLR(CPU_FTR_POWER9_DD2_1) LOAD_REG_ADDRBASE(r5,pnv_first_deep_stop_state) ld r4,ADDROFF(pnv_first_deep_stop_state)(r5) -BEGIN_FTR_SECTION_NESTED(71) - /* - * Assume that we are waking up from the state - * same as the Requested Level (RL) in the PSSCR - * which are Bits 60-63 - */ - ld r5,PACA_REQ_PSSCR(r13) - rldicl r5,r5,0,60 -FTR_SECTION_ELSE_NESTED(71) /* * 0-3 bits correspond to Power-Saving Level Status * which indicates the idle state we are waking up from */ mfspr r5, SPRN_PSSCR rldicl r5,r5,4,60 -ALT_FTR_SECTION_END_NESTED_IFSET(CPU_FTR_POWER9_DD1, 71) li r0, 0 /* clear requested_psscr to say we're awake */ std r0, PACA_REQ_PSSCR(r13) cmpd cr4,r5,r4 diff --git a/arch/powerpc/kernel/idle_e500.S b/arch/powerpc/kernel/idle_e500.S index b9b6ef510be1..583e55ac7d26 100644 --- a/arch/powerpc/kernel/idle_e500.S +++ b/arch/powerpc/kernel/idle_e500.S @@ -17,6 +17,7 @@ #include <asm/thread_info.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/feature-fixups.h> .text diff --git a/arch/powerpc/kernel/idle_power4.S b/arch/powerpc/kernel/idle_power4.S index 08faa93755f9..dd7471fe20bd 100644 --- a/arch/powerpc/kernel/idle_power4.S +++ b/arch/powerpc/kernel/idle_power4.S @@ -16,6 +16,7 @@ #include <asm/asm-offsets.h> #include <asm/irqflags.h> #include <asm/hw_irq.h> +#include <asm/feature-fixups.h> #undef DEBUG diff --git a/arch/powerpc/kernel/irq.c b/arch/powerpc/kernel/irq.c index 0682fef1f385..916ddc4aac44 100644 --- a/arch/powerpc/kernel/irq.c +++ b/arch/powerpc/kernel/irq.c @@ -145,8 +145,20 @@ notrace unsigned int __check_irq_replay(void) trace_hardirqs_on(); trace_hardirqs_off(); + /* + * We are always hard disabled here, but PACA_IRQ_HARD_DIS may + * not be set, which means interrupts have only just been hard + * disabled as part of the local_irq_restore or interrupt return + * code. In that case, skip the decrementr check becaus it's + * expensive to read the TB. + * + * HARD_DIS then gets cleared here, but it's reconciled later. + * Either local_irq_disable will replay the interrupt and that + * will reconcile state like other hard interrupts. Or interrupt + * retur will replay the interrupt and in that case it sets + * PACA_IRQ_HARD_DIS by hand (see comments in entry_64.S). + */ if (happened & PACA_IRQ_HARD_DIS) { - /* Clear bit 0 which we wouldn't clear otherwise */ local_paca->irq_happened &= ~PACA_IRQ_HARD_DIS; /* @@ -248,24 +260,33 @@ notrace void arch_local_irq_restore(unsigned long mask) * cannot have preempted. */ irq_happened = get_irq_happened(); - if (!irq_happened) + if (!irq_happened) { + /* + * FIXME. Here we'd like to be able to do: + * + * #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG + * WARN_ON(!(mfmsr() & MSR_EE)); + * #endif + * + * But currently it hits in a few paths, we should fix those and + * enable the warning. + */ return; + } /* * We need to hard disable to get a trusted value from * __check_irq_replay(). We also need to soft-disable * again to avoid warnings in there due to the use of * per-cpu variables. - * - * We know that if the value in irq_happened is exactly 0x01 - * then we are already hard disabled (there are other less - * common cases that we'll ignore for now), so we skip the - * (expensive) mtmsrd. */ - if (unlikely(irq_happened != PACA_IRQ_HARD_DIS)) + if (!(irq_happened & PACA_IRQ_HARD_DIS)) { +#ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG + WARN_ON(!(mfmsr() & MSR_EE)); +#endif __hard_irq_disable(); #ifdef CONFIG_PPC_IRQ_SOFT_MASK_DEBUG - else { + } else { /* * We should already be hard disabled here. We had bugs * where that wasn't the case so let's dbl check it and @@ -274,8 +295,8 @@ notrace void arch_local_irq_restore(unsigned long mask) */ if (WARN_ON(mfmsr() & MSR_EE)) __hard_irq_disable(); - } #endif + } irq_soft_mask_set(IRQS_ALL_DISABLED); trace_hardirqs_off(); diff --git a/arch/powerpc/kernel/kvm_emul.S b/arch/powerpc/kernel/kvm_emul.S index e100ff324a85..c005088f6c9c 100644 --- a/arch/powerpc/kernel/kvm_emul.S +++ b/arch/powerpc/kernel/kvm_emul.S @@ -23,6 +23,7 @@ #include <asm/reg.h> #include <asm/page.h> #include <asm/asm-offsets.h> +#include <asm/asm-compat.h> #define KVM_MAGIC_PAGE (-4096) diff --git a/arch/powerpc/kernel/l2cr_6xx.S b/arch/powerpc/kernel/l2cr_6xx.S index 6408f09dbbd9..6e7dbb7d527c 100644 --- a/arch/powerpc/kernel/l2cr_6xx.S +++ b/arch/powerpc/kernel/l2cr_6xx.S @@ -45,6 +45,7 @@ #include <asm/ppc_asm.h> #include <asm/cache.h> #include <asm/page.h> +#include <asm/feature-fixups.h> /* Usage: diff --git a/arch/powerpc/kernel/machine_kexec.c b/arch/powerpc/kernel/machine_kexec.c index 936c7e2d421e..63f5a9311a29 100644 --- a/arch/powerpc/kernel/machine_kexec.c +++ b/arch/powerpc/kernel/machine_kexec.c @@ -17,6 +17,7 @@ #include <linux/irq.h> #include <linux/ftrace.h> +#include <asm/kdump.h> #include <asm/machdep.h> #include <asm/pgalloc.h> #include <asm/prom.h> @@ -188,7 +189,12 @@ void __init reserve_crashkernel(void) (unsigned long)(crashk_res.start >> 20), (unsigned long)(memblock_phys_mem_size() >> 20)); - memblock_reserve(crashk_res.start, crash_size); + if (!memblock_is_region_memory(crashk_res.start, crash_size) || + memblock_reserve(crashk_res.start, crash_size)) { + pr_err("Failed to reserve memory for crashkernel!\n"); + crashk_res.start = crashk_res.end = 0; + return; + } } int overlaps_crashkernel(unsigned long start, unsigned long size) diff --git a/arch/powerpc/kernel/machine_kexec_file_64.c b/arch/powerpc/kernel/machine_kexec_file_64.c index 0bd23dc789a4..c77e95e9b384 100644 --- a/arch/powerpc/kernel/machine_kexec_file_64.c +++ b/arch/powerpc/kernel/machine_kexec_file_64.c @@ -269,18 +269,14 @@ int setup_new_fdt(const struct kimage *image, void *fdt, ret = fdt_setprop_u64(fdt, chosen_node, "linux,initrd-start", initrd_load_addr); - if (ret < 0) { - pr_err("Error setting up the new device tree.\n"); - return -EINVAL; - } + if (ret < 0) + goto err; /* initrd-end is the first address after the initrd image. */ ret = fdt_setprop_u64(fdt, chosen_node, "linux,initrd-end", initrd_load_addr + initrd_len); - if (ret < 0) { - pr_err("Error setting up the new device tree.\n"); - return -EINVAL; - } + if (ret < 0) + goto err; ret = fdt_add_mem_rsv(fdt, initrd_load_addr, initrd_len); if (ret) { @@ -292,10 +288,8 @@ int setup_new_fdt(const struct kimage *image, void *fdt, if (cmdline != NULL) { ret = fdt_setprop_string(fdt, chosen_node, "bootargs", cmdline); - if (ret < 0) { - pr_err("Error setting up the new device tree.\n"); - return -EINVAL; - } + if (ret < 0) + goto err; } else { ret = fdt_delprop(fdt, chosen_node, "bootargs"); if (ret && ret != -FDT_ERR_NOTFOUND) { @@ -311,10 +305,12 @@ int setup_new_fdt(const struct kimage *image, void *fdt, } ret = fdt_setprop(fdt, chosen_node, "linux,booted-from-kexec", NULL, 0); - if (ret) { - pr_err("Error setting up the new device tree.\n"); - return -EINVAL; - } + if (ret) + goto err; return 0; + +err: + pr_err("Error setting up the new device tree.\n"); + return -EINVAL; } diff --git a/arch/powerpc/kernel/mce_power.c b/arch/powerpc/kernel/mce_power.c index 38c5b4764bfe..3497c8329c1d 100644 --- a/arch/powerpc/kernel/mce_power.c +++ b/arch/powerpc/kernel/mce_power.c @@ -62,11 +62,8 @@ static unsigned long addr_to_pfn(struct pt_regs *regs, unsigned long addr) #ifdef CONFIG_PPC_BOOK3S_64 static void flush_and_reload_slb(void) { - struct slb_shadow *slb; - unsigned long i, n; - /* Invalidate all SLBs */ - asm volatile("slbmte %0,%0; slbia" : : "r" (0)); + slb_flush_all_realmode(); #ifdef CONFIG_KVM_BOOK3S_HANDLER /* @@ -76,22 +73,17 @@ static void flush_and_reload_slb(void) if (get_paca()->kvm_hstate.in_guest) return; #endif - - /* For host kernel, reload the SLBs from shadow SLB buffer. */ - slb = get_slb_shadow(); - if (!slb) + if (early_radix_enabled()) return; - n = min_t(u32, be32_to_cpu(slb->persistent), SLB_MIN_SIZE); - - /* Load up the SLB entries from shadow SLB */ - for (i = 0; i < n; i++) { - unsigned long rb = be64_to_cpu(slb->save_area[i].esid); - unsigned long rs = be64_to_cpu(slb->save_area[i].vsid); + /* + * This probably shouldn't happen, but it may be possible it's + * called in early boot before SLB shadows are allocated. + */ + if (!get_slb_shadow()) + return; - rb = (rb & ~0xFFFul) | i; - asm volatile("slbmte %0,%1" : : "r" (rs), "r" (rb)); - } + slb_restore_bolted_realmode(); } #endif @@ -257,12 +249,12 @@ static const struct mce_derror_table mce_p7_derror_table[] = { { 0x00000400, true, MCE_ERROR_TYPE_TLB, MCE_TLB_ERROR_MULTIHIT, MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, +{ 0x00000080, true, + MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, /* Before PARITY */ + MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, { 0x00000100, true, MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY, MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, -{ 0x00000080, true, - MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, - MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, { 0x00000040, true, MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_INDETERMINATE, /* BOTH */ MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, @@ -290,12 +282,12 @@ static const struct mce_derror_table mce_p8_derror_table[] = { { 0x00000200, true, MCE_ERROR_TYPE_ERAT, MCE_ERAT_ERROR_MULTIHIT, /* SECONDARY ERAT */ MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, +{ 0x00000080, true, + MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, /* Before PARITY */ + MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, { 0x00000100, true, MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY, MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, -{ 0x00000080, true, - MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, - MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, { 0, false, 0, 0, 0, 0 } }; static const struct mce_derror_table mce_p9_derror_table[] = { @@ -320,12 +312,12 @@ static const struct mce_derror_table mce_p9_derror_table[] = { { 0x00000200, false, MCE_ERROR_TYPE_USER, MCE_USER_ERROR_TLBIE, MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, +{ 0x00000080, true, + MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, /* Before PARITY */ + MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, { 0x00000100, true, MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_PARITY, MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, -{ 0x00000080, true, - MCE_ERROR_TYPE_SLB, MCE_SLB_ERROR_MULTIHIT, - MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, { 0x00000040, true, MCE_ERROR_TYPE_RA, MCE_RA_ERROR_LOAD, MCE_INITIATOR_CPU, MCE_SEV_ERROR_SYNC, }, diff --git a/arch/powerpc/kernel/misc_32.S b/arch/powerpc/kernel/misc_32.S index 3f7a9a2d2435..695b24a2d954 100644 --- a/arch/powerpc/kernel/misc_32.S +++ b/arch/powerpc/kernel/misc_32.S @@ -34,6 +34,7 @@ #include <asm/bug.h> #include <asm/ptrace.h> #include <asm/export.h> +#include <asm/feature-fixups.h> .text diff --git a/arch/powerpc/kernel/misc_64.S b/arch/powerpc/kernel/misc_64.S index fa267e94090a..262ba9481781 100644 --- a/arch/powerpc/kernel/misc_64.S +++ b/arch/powerpc/kernel/misc_64.S @@ -28,6 +28,7 @@ #include <asm/ptrace.h> #include <asm/mmu.h> #include <asm/export.h> +#include <asm/feature-fixups.h> .text diff --git a/arch/powerpc/kernel/module.c b/arch/powerpc/kernel/module.c index 1b3c6835e730..77371c9ef3d8 100644 --- a/arch/powerpc/kernel/module.c +++ b/arch/powerpc/kernel/module.c @@ -72,13 +72,15 @@ int module_finalize(const Elf_Ehdr *hdr, do_feature_fixups(powerpc_firmware_features, (void *)sect->sh_addr, (void *)sect->sh_addr + sect->sh_size); +#endif /* CONFIG_PPC64 */ +#ifdef CONFIG_PPC_BARRIER_NOSPEC sect = find_section(hdr, sechdrs, "__spec_barrier_fixup"); if (sect != NULL) do_barrier_nospec_fixups_range(barrier_nospec_enabled, (void *)sect->sh_addr, (void *)sect->sh_addr + sect->sh_size); -#endif +#endif /* CONFIG_PPC_BARRIER_NOSPEC */ sect = find_section(hdr, sechdrs, "__lwsync_fixup"); if (sect != NULL) diff --git a/arch/powerpc/kernel/pci-common.c b/arch/powerpc/kernel/pci-common.c index 471aac313b89..88e4f69a09e5 100644 --- a/arch/powerpc/kernel/pci-common.c +++ b/arch/powerpc/kernel/pci-common.c @@ -368,9 +368,6 @@ static int pci_read_irq_line(struct pci_dev *pci_dev) pr_debug("PCI: Try to map irq for %s...\n", pci_name(pci_dev)); -#ifdef DEBUG - memset(&oirq, 0xff, sizeof(oirq)); -#endif /* Try to get a mapping from the device-tree */ virq = of_irq_parse_and_map_pci(pci_dev, 0, 0); if (virq <= 0) { diff --git a/arch/powerpc/kernel/ppc_save_regs.S b/arch/powerpc/kernel/ppc_save_regs.S index 8afbe213d729..6d1b42ee797c 100644 --- a/arch/powerpc/kernel/ppc_save_regs.S +++ b/arch/powerpc/kernel/ppc_save_regs.S @@ -12,6 +12,7 @@ #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/ptrace.h> +#include <asm/asm-compat.h> /* * Grab the register values as they are now. diff --git a/arch/powerpc/kernel/process.c b/arch/powerpc/kernel/process.c index 9ef4aea9fffe..913c5725cdb2 100644 --- a/arch/powerpc/kernel/process.c +++ b/arch/powerpc/kernel/process.c @@ -583,6 +583,7 @@ static void save_all(struct task_struct *tsk) __giveup_spe(tsk); msr_check_and_clear(msr_all_available); + thread_pkey_regs_save(&tsk->thread); } void flush_all_to_thread(struct task_struct *tsk) @@ -716,6 +717,13 @@ void switch_booke_debug_regs(struct debug_reg *new_debug) EXPORT_SYMBOL_GPL(switch_booke_debug_regs); #else /* !CONFIG_PPC_ADV_DEBUG_REGS */ #ifndef CONFIG_HAVE_HW_BREAKPOINT +static void set_breakpoint(struct arch_hw_breakpoint *brk) +{ + preempt_disable(); + __set_breakpoint(brk); + preempt_enable(); +} + static void set_debug_reg_defaults(struct thread_struct *thread) { thread->hw_brk.address = 0; @@ -828,13 +836,6 @@ void __set_breakpoint(struct arch_hw_breakpoint *brk) WARN_ON_ONCE(1); } -void set_breakpoint(struct arch_hw_breakpoint *brk) -{ - preempt_disable(); - __set_breakpoint(brk); - preempt_enable(); -} - /* Check if we have DAWR or DABR hardware */ bool ppc_breakpoint_available(void) { @@ -866,8 +867,7 @@ static inline bool tm_enabled(struct task_struct *tsk) return tsk && tsk->thread.regs && (tsk->thread.regs->msr & MSR_TM); } -static void tm_reclaim_thread(struct thread_struct *thr, - struct thread_info *ti, uint8_t cause) +static void tm_reclaim_thread(struct thread_struct *thr, uint8_t cause) { /* * Use the current MSR TM suspended bit to track if we have @@ -914,7 +914,7 @@ static void tm_reclaim_thread(struct thread_struct *thr, void tm_reclaim_current(uint8_t cause) { tm_enable(); - tm_reclaim_thread(¤t->thread, current_thread_info(), cause); + tm_reclaim_thread(¤t->thread, cause); } static inline void tm_reclaim_task(struct task_struct *tsk) @@ -945,7 +945,7 @@ static inline void tm_reclaim_task(struct task_struct *tsk) thr->regs->ccr, thr->regs->msr, thr->regs->trap); - tm_reclaim_thread(thr, task_thread_info(tsk), TM_CAUSE_RESCHED); + tm_reclaim_thread(thr, TM_CAUSE_RESCHED); TM_DEBUG("--- tm_reclaim on pid %d complete\n", tsk->pid); @@ -1250,17 +1250,9 @@ struct task_struct *__switch_to(struct task_struct *prev, * mappings. If the new process has the foreign real address * mappings, we must issue a cp_abort to clear any state and * prevent snooping, corruption or a covert channel. - * - * DD1 allows paste into normal system memory so we do an - * unpaired copy, rather than cp_abort, to clear the buffer, - * since cp_abort is quite expensive. */ - if (current_thread_info()->task->thread.used_vas) { + if (current_thread_info()->task->thread.used_vas) asm volatile(PPC_CP_ABORT); - } else if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { - asm volatile(PPC_COPY(%0, %1) - : : "r"(dummy_copy_buffer), "r"(0)); - } } #endif /* CONFIG_PPC_BOOK3S_64 */ @@ -1307,6 +1299,38 @@ static void show_instructions(struct pt_regs *regs) pr_cont("\n"); } +void show_user_instructions(struct pt_regs *regs) +{ + unsigned long pc; + int i; + + pc = regs->nip - (instructions_to_print * 3 / 4 * sizeof(int)); + + pr_info("%s[%d]: code: ", current->comm, current->pid); + + for (i = 0; i < instructions_to_print; i++) { + int instr; + + if (!(i % 8) && (i > 0)) { + pr_cont("\n"); + pr_info("%s[%d]: code: ", current->comm, current->pid); + } + + if (probe_kernel_address((unsigned int __user *)pc, instr)) { + pr_cont("XXXXXXXX "); + } else { + if (regs->nip == pc) + pr_cont("<%08x> ", instr); + else + pr_cont("%08x ", instr); + } + + pc += sizeof(int); + } + + pr_cont("\n"); +} + struct regbit { unsigned long bit; const char *name; diff --git a/arch/powerpc/kernel/prom.c b/arch/powerpc/kernel/prom.c index 05e7fb47a7a4..c4d7078e5295 100644 --- a/arch/powerpc/kernel/prom.c +++ b/arch/powerpc/kernel/prom.c @@ -23,7 +23,6 @@ #include <linux/spinlock.h> #include <linux/types.h> #include <linux/pci.h> -#include <linux/stringify.h> #include <linux/delay.h> #include <linux/initrd.h> #include <linux/bitops.h> @@ -440,6 +439,29 @@ static int __init early_init_dt_scan_chosen_ppc(unsigned long node, return 1; } +/* + * Compare the range against max mem limit and update + * size if it cross the limit. + */ + +#ifdef CONFIG_SPARSEMEM +static bool validate_mem_limit(u64 base, u64 *size) +{ + u64 max_mem = 1UL << (MAX_PHYSMEM_BITS); + + if (base >= max_mem) + return false; + if ((base + *size) > max_mem) + *size = max_mem - base; + return true; +} +#else +static bool validate_mem_limit(u64 base, u64 *size) +{ + return true; +} +#endif + #ifdef CONFIG_PPC_PSERIES /* * Interpret the ibm dynamic reconfiguration memory LMBs. @@ -494,7 +516,8 @@ static void __init early_init_drmem_lmb(struct drmem_lmb *lmb, } DBG("Adding: %llx -> %llx\n", base, size); - memblock_add(base, size); + if (validate_mem_limit(base, &size)) + memblock_add(base, size); } while (--rngs); } #endif /* CONFIG_PPC_PSERIES */ @@ -548,8 +571,10 @@ void __init early_init_dt_add_memory_arch(u64 base, u64 size) } /* Add the chunk to the MEMBLOCK list */ - if (add_mem_to_memblock) - memblock_add(base, size); + if (add_mem_to_memblock) { + if (validate_mem_limit(base, &size)) + memblock_add(base, size); + } } static void __init early_reserve_mem_dt(void) diff --git a/arch/powerpc/kernel/prom_init.c b/arch/powerpc/kernel/prom_init.c index 5425dd3d6a9f..9b38a2e5dd35 100644 --- a/arch/powerpc/kernel/prom_init.c +++ b/arch/powerpc/kernel/prom_init.c @@ -27,7 +27,6 @@ #include <linux/types.h> #include <linux/pci.h> #include <linux/proc_fs.h> -#include <linux/stringify.h> #include <linux/delay.h> #include <linux/initrd.h> #include <linux/bitops.h> @@ -629,7 +628,7 @@ static void __init early_cmdline_parse(void) const char *opt; char *p; - int l = 0; + int l __maybe_unused = 0; prom_cmd_line[0] = 0; p = prom_cmd_line; @@ -1422,7 +1421,10 @@ static void __init reserve_mem(u64 base, u64 size) static void __init prom_init_mem(void) { phandle node; - char *path, type[64]; +#ifdef DEBUG_PROM + char *path; +#endif + char type[64]; unsigned int plen; cell_t *p, *endp; __be32 val; @@ -1443,7 +1445,9 @@ static void __init prom_init_mem(void) prom_debug("root_size_cells: %x\n", rsc); prom_debug("scanning memory:\n"); +#ifdef DEBUG_PROM path = prom_scratch; +#endif for (node = 0; prom_next_node(&node); ) { type[0] = 0; @@ -2102,8 +2106,6 @@ static void __init prom_init_stdout(void) stdout_node = call_prom("instance-to-package", 1, 1, prom.stdout); if (stdout_node != PROM_ERROR) { val = cpu_to_be32(stdout_node); - prom_setprop(prom.chosen, "/chosen", "linux,stdout-package", - &val, sizeof(val)); /* If it's a display, note it */ memset(type, 0, sizeof(type)); diff --git a/arch/powerpc/kernel/security.c b/arch/powerpc/kernel/security.c index a8b277362931..f6f469fc4073 100644 --- a/arch/powerpc/kernel/security.c +++ b/arch/powerpc/kernel/security.c @@ -8,6 +8,8 @@ #include <linux/device.h> #include <linux/seq_buf.h> +#include <asm/asm-prototypes.h> +#include <asm/code-patching.h> #include <asm/debugfs.h> #include <asm/security_features.h> #include <asm/setup.h> @@ -15,7 +17,15 @@ unsigned long powerpc_security_features __read_mostly = SEC_FTR_DEFAULT; +enum count_cache_flush_type { + COUNT_CACHE_FLUSH_NONE = 0x1, + COUNT_CACHE_FLUSH_SW = 0x2, + COUNT_CACHE_FLUSH_HW = 0x4, +}; +static enum count_cache_flush_type count_cache_flush_type; + bool barrier_nospec_enabled; +static bool no_nospec; static void enable_barrier_nospec(bool enable) { @@ -42,8 +52,17 @@ void setup_barrier_nospec(void) enable = security_ftr_enabled(SEC_FTR_FAVOUR_SECURITY) && security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR); - enable_barrier_nospec(enable); + if (!no_nospec) + enable_barrier_nospec(enable); +} + +static int __init handle_nospectre_v1(char *p) +{ + no_nospec = true; + + return 0; } +early_param("nospectre_v1", handle_nospectre_v1); #ifdef CONFIG_DEBUG_FS static int barrier_nospec_set(void *data, u64 val) @@ -82,6 +101,7 @@ static __init int barrier_nospec_debugfs_init(void) device_initcall(barrier_nospec_debugfs_init); #endif /* CONFIG_DEBUG_FS */ +#ifdef CONFIG_PPC_BOOK3S_64 ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, char *buf) { bool thread_priv; @@ -114,51 +134,72 @@ ssize_t cpu_show_meltdown(struct device *dev, struct device_attribute *attr, cha return sprintf(buf, "Vulnerable\n"); } +#endif ssize_t cpu_show_spectre_v1(struct device *dev, struct device_attribute *attr, char *buf) { - if (!security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR)) - return sprintf(buf, "Not affected\n"); + struct seq_buf s; - if (barrier_nospec_enabled) - return sprintf(buf, "Mitigation: __user pointer sanitization\n"); + seq_buf_init(&s, buf, PAGE_SIZE - 1); - return sprintf(buf, "Vulnerable\n"); + if (security_ftr_enabled(SEC_FTR_BNDS_CHK_SPEC_BAR)) { + if (barrier_nospec_enabled) + seq_buf_printf(&s, "Mitigation: __user pointer sanitization"); + else + seq_buf_printf(&s, "Vulnerable"); + + if (security_ftr_enabled(SEC_FTR_SPEC_BAR_ORI31)) + seq_buf_printf(&s, ", ori31 speculation barrier enabled"); + + seq_buf_printf(&s, "\n"); + } else + seq_buf_printf(&s, "Not affected\n"); + + return s.len; } ssize_t cpu_show_spectre_v2(struct device *dev, struct device_attribute *attr, char *buf) { - bool bcs, ccd, ori; struct seq_buf s; + bool bcs, ccd; seq_buf_init(&s, buf, PAGE_SIZE - 1); bcs = security_ftr_enabled(SEC_FTR_BCCTRL_SERIALISED); ccd = security_ftr_enabled(SEC_FTR_COUNT_CACHE_DISABLED); - ori = security_ftr_enabled(SEC_FTR_SPEC_BAR_ORI31); - if (bcs || ccd) { + if (bcs || ccd || count_cache_flush_type != COUNT_CACHE_FLUSH_NONE) { + bool comma = false; seq_buf_printf(&s, "Mitigation: "); - if (bcs) + if (bcs) { seq_buf_printf(&s, "Indirect branch serialisation (kernel only)"); + comma = true; + } + + if (ccd) { + if (comma) + seq_buf_printf(&s, ", "); + seq_buf_printf(&s, "Indirect branch cache disabled"); + comma = true; + } - if (bcs && ccd) + if (comma) seq_buf_printf(&s, ", "); - if (ccd) - seq_buf_printf(&s, "Indirect branch cache disabled"); + seq_buf_printf(&s, "Software count cache flush"); + + if (count_cache_flush_type == COUNT_CACHE_FLUSH_HW) + seq_buf_printf(&s, "(hardware accelerated)"); } else seq_buf_printf(&s, "Vulnerable"); - if (ori) - seq_buf_printf(&s, ", ori31 speculation barrier enabled"); - seq_buf_printf(&s, "\n"); return s.len; } +#ifdef CONFIG_PPC_BOOK3S_64 /* * Store-forwarding barrier support. */ @@ -306,3 +347,71 @@ static __init int stf_barrier_debugfs_init(void) } device_initcall(stf_barrier_debugfs_init); #endif /* CONFIG_DEBUG_FS */ + +static void toggle_count_cache_flush(bool enable) +{ + if (!enable || !security_ftr_enabled(SEC_FTR_FLUSH_COUNT_CACHE)) { + patch_instruction_site(&patch__call_flush_count_cache, PPC_INST_NOP); + count_cache_flush_type = COUNT_CACHE_FLUSH_NONE; + pr_info("count-cache-flush: software flush disabled.\n"); + return; + } + + patch_branch_site(&patch__call_flush_count_cache, + (u64)&flush_count_cache, BRANCH_SET_LINK); + + if (!security_ftr_enabled(SEC_FTR_BCCTR_FLUSH_ASSIST)) { + count_cache_flush_type = COUNT_CACHE_FLUSH_SW; + pr_info("count-cache-flush: full software flush sequence enabled.\n"); + return; + } + + patch_instruction_site(&patch__flush_count_cache_return, PPC_INST_BLR); + count_cache_flush_type = COUNT_CACHE_FLUSH_HW; + pr_info("count-cache-flush: hardware assisted flush sequence enabled\n"); +} + +void setup_count_cache_flush(void) +{ + toggle_count_cache_flush(true); +} + +#ifdef CONFIG_DEBUG_FS +static int count_cache_flush_set(void *data, u64 val) +{ + bool enable; + + if (val == 1) + enable = true; + else if (val == 0) + enable = false; + else + return -EINVAL; + + toggle_count_cache_flush(enable); + + return 0; +} + +static int count_cache_flush_get(void *data, u64 *val) +{ + if (count_cache_flush_type == COUNT_CACHE_FLUSH_NONE) + *val = 0; + else + *val = 1; + + return 0; +} + +DEFINE_SIMPLE_ATTRIBUTE(fops_count_cache_flush, count_cache_flush_get, + count_cache_flush_set, "%llu\n"); + +static __init int count_cache_flush_debugfs_init(void) +{ + debugfs_create_file("count_cache_flush", 0600, powerpc_debugfs_root, + NULL, &fops_count_cache_flush); + return 0; +} +device_initcall(count_cache_flush_debugfs_init); +#endif /* CONFIG_DEBUG_FS */ +#endif /* CONFIG_PPC_BOOK3S_64 */ diff --git a/arch/powerpc/kernel/setup-common.c b/arch/powerpc/kernel/setup-common.c index 40b44bb53a4e..93fa0c99681e 100644 --- a/arch/powerpc/kernel/setup-common.c +++ b/arch/powerpc/kernel/setup-common.c @@ -972,6 +972,8 @@ void __init setup_arch(char **cmdline_p) if (ppc_md.setup_arch) ppc_md.setup_arch(); + setup_barrier_nospec(); + paging_init(); /* Initialize the MMU context management stuff. */ diff --git a/arch/powerpc/kernel/setup_32.c b/arch/powerpc/kernel/setup_32.c index 74457485574b..8c507be12c3c 100644 --- a/arch/powerpc/kernel/setup_32.c +++ b/arch/powerpc/kernel/setup_32.c @@ -40,6 +40,10 @@ #include <asm/code-patching.h> #include <asm/cpu_has_feature.h> #include <asm/asm-prototypes.h> +#include <asm/kdump.h> +#include <asm/feature-fixups.h> + +#include "setup.h" #define DBG(fmt...) @@ -95,11 +99,10 @@ notrace unsigned long __init early_init(unsigned long dt_ptr) * We do the initial parsing of the flat device-tree and prepares * for the MMU to be fully initialized. */ -extern unsigned int memset_nocache_branch; /* Insn to be replaced by NOP */ - notrace void __init machine_init(u64 dt_ptr) { - unsigned int *addr = &memset_nocache_branch; + unsigned int *addr = (unsigned int *)((unsigned long)&patch__memset_nocache + + patch__memset_nocache); unsigned long insn; /* Configure static keys first, now that we're relocated. */ @@ -108,7 +111,7 @@ notrace void __init machine_init(u64 dt_ptr) /* Enable early debugging if any specified (see udbg.h) */ udbg_early_init(); - patch_instruction((unsigned int *)&memcpy, PPC_INST_NOP); + patch_instruction_site(&patch__memcpy_nocache, PPC_INST_NOP); insn = create_cond_branch(addr, branch_target(addr), 0x820000); patch_instruction(addr, insn); /* replace b by bne cr0 */ diff --git a/arch/powerpc/kernel/setup_64.c b/arch/powerpc/kernel/setup_64.c index 225bc5f91049..6a501b25dd85 100644 --- a/arch/powerpc/kernel/setup_64.c +++ b/arch/powerpc/kernel/setup_64.c @@ -68,6 +68,7 @@ #include <asm/opal.h> #include <asm/cputhreads.h> #include <asm/hw_irq.h> +#include <asm/feature-fixups.h> #include "setup.h" diff --git a/arch/powerpc/kernel/smp.c b/arch/powerpc/kernel/smp.c index 4794d6b4f4d2..b19d832ef386 100644 --- a/arch/powerpc/kernel/smp.c +++ b/arch/powerpc/kernel/smp.c @@ -423,7 +423,8 @@ int smp_handle_nmi_ipi(struct pt_regs *regs) fn(regs); nmi_ipi_lock(); - nmi_ipi_busy_count--; + if (nmi_ipi_busy_count > 1) /* Can race with caller time-out */ + nmi_ipi_busy_count--; out: nmi_ipi_unlock_end(&flags); @@ -448,29 +449,11 @@ static void do_smp_send_nmi_ipi(int cpu, bool safe) } } -void smp_flush_nmi_ipi(u64 delay_us) -{ - unsigned long flags; - - nmi_ipi_lock_start(&flags); - while (nmi_ipi_busy_count) { - nmi_ipi_unlock_end(&flags); - udelay(1); - if (delay_us) { - delay_us--; - if (!delay_us) - return; - } - nmi_ipi_lock_start(&flags); - } - nmi_ipi_unlock_end(&flags); -} - /* * - cpu is the target CPU (must not be this CPU), or NMI_IPI_ALL_OTHERS. * - fn is the target callback function. * - delay_us > 0 is the delay before giving up waiting for targets to - * enter the handler, == 0 specifies indefinite delay. + * complete executing the handler, == 0 specifies indefinite delay. */ int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us, bool safe) { @@ -507,8 +490,23 @@ int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us, bool do_smp_send_nmi_ipi(cpu, safe); + nmi_ipi_lock(); + /* nmi_ipi_busy_count is held here, so unlock/lock is okay */ while (!cpumask_empty(&nmi_ipi_pending_mask)) { + nmi_ipi_unlock(); udelay(1); + nmi_ipi_lock(); + if (delay_us) { + delay_us--; + if (!delay_us) + break; + } + } + + while (nmi_ipi_busy_count > 1) { + nmi_ipi_unlock(); + udelay(1); + nmi_ipi_lock(); if (delay_us) { delay_us--; if (!delay_us) @@ -516,12 +514,17 @@ int __smp_send_nmi_ipi(int cpu, void (*fn)(struct pt_regs *), u64 delay_us, bool } } - nmi_ipi_lock(); if (!cpumask_empty(&nmi_ipi_pending_mask)) { - /* Could not gather all CPUs */ + /* Timeout waiting for CPUs to call smp_handle_nmi_ipi */ ret = 0; cpumask_clear(&nmi_ipi_pending_mask); } + if (nmi_ipi_busy_count > 1) { + /* Timeout waiting for CPUs to execute fn */ + ret = 0; + nmi_ipi_busy_count = 1; + } + nmi_ipi_busy_count--; nmi_ipi_unlock_end(&flags); @@ -597,7 +600,8 @@ static void nmi_stop_this_cpu(struct pt_regs *regs) * IRQs are already hard disabled by the smp_handle_nmi_ipi. */ nmi_ipi_lock(); - nmi_ipi_busy_count--; + if (nmi_ipi_busy_count > 1) + nmi_ipi_busy_count--; nmi_ipi_unlock(); spin_begin(); diff --git a/arch/powerpc/kernel/swsusp_32.S b/arch/powerpc/kernel/swsusp_32.S index 34b73a262709..7a919e9a3400 100644 --- a/arch/powerpc/kernel/swsusp_32.S +++ b/arch/powerpc/kernel/swsusp_32.S @@ -7,6 +7,7 @@ #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/mmu.h> +#include <asm/feature-fixups.h> /* * Structure for storing CPU registers on the save area. diff --git a/arch/powerpc/kernel/swsusp_asm64.S b/arch/powerpc/kernel/swsusp_asm64.S index 82d8aae81c6a..f83bf6f72cb0 100644 --- a/arch/powerpc/kernel/swsusp_asm64.S +++ b/arch/powerpc/kernel/swsusp_asm64.S @@ -13,6 +13,7 @@ #include <asm/thread_info.h> #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> +#include <asm/feature-fixups.h> /* * Structure for storing CPU registers on the save area. diff --git a/arch/powerpc/kernel/tm.S b/arch/powerpc/kernel/tm.S index ff12f47a96b6..6bffbc5affe7 100644 --- a/arch/powerpc/kernel/tm.S +++ b/arch/powerpc/kernel/tm.S @@ -13,6 +13,7 @@ #include <asm/reg.h> #include <asm/bug.h> #include <asm/export.h> +#include <asm/feature-fixups.h> #ifdef CONFIG_VSX /* See fpu.S, this is borrowed from there */ @@ -312,8 +313,8 @@ _GLOBAL(tm_reclaim) blr - /* void __tm_recheckpoint(struct thread_struct *thread, - * unsigned long orig_msr) + /* + * void __tm_recheckpoint(struct thread_struct *thread) * - Restore the checkpointed register state saved by tm_reclaim * when we switch_to a process. * diff --git a/arch/powerpc/kernel/traps.c b/arch/powerpc/kernel/traps.c index 0e17dcb48720..070e96f1773a 100644 --- a/arch/powerpc/kernel/traps.c +++ b/arch/powerpc/kernel/traps.c @@ -70,6 +70,7 @@ #include <asm/hmi.h> #include <sysdev/fsl_pci.h> #include <asm/kprobes.h> +#include <asm/stacktrace.h> #if defined(CONFIG_DEBUGGER) || defined(CONFIG_KEXEC_CORE) int (*__debugger)(struct pt_regs *regs) __read_mostly; @@ -96,6 +97,19 @@ EXPORT_SYMBOL(__debugger_fault_handler); #define TM_DEBUG(x...) do { } while(0) #endif +static const char *signame(int signr) +{ + switch (signr) { + case SIGBUS: return "bus error"; + case SIGFPE: return "floating point exception"; + case SIGILL: return "illegal instruction"; + case SIGSEGV: return "segfault"; + case SIGTRAP: return "unhandled trap"; + } + + return "unknown signal"; +} + /* * Trap & Exception support */ @@ -301,26 +315,44 @@ void user_single_step_siginfo(struct task_struct *tsk, info->si_addr = (void __user *)regs->nip; } +static bool show_unhandled_signals_ratelimited(void) +{ + static DEFINE_RATELIMIT_STATE(rs, DEFAULT_RATELIMIT_INTERVAL, + DEFAULT_RATELIMIT_BURST); + return show_unhandled_signals && __ratelimit(&rs); +} + +static void show_signal_msg(int signr, struct pt_regs *regs, int code, + unsigned long addr) +{ + if (!show_unhandled_signals_ratelimited()) + return; + + if (!unhandled_signal(current, signr)) + return; + + pr_info("%s[%d]: %s (%d) at %lx nip %lx lr %lx code %x", + current->comm, current->pid, signame(signr), signr, + addr, regs->nip, regs->link, code); + + print_vma_addr(KERN_CONT " in ", regs->nip); + + pr_cont("\n"); + + show_user_instructions(regs); +} void _exception_pkey(int signr, struct pt_regs *regs, int code, - unsigned long addr, int key) + unsigned long addr, int key) { siginfo_t info; - const char fmt32[] = KERN_INFO "%s[%d]: unhandled signal %d " \ - "at %08lx nip %08lx lr %08lx code %x\n"; - const char fmt64[] = KERN_INFO "%s[%d]: unhandled signal %d " \ - "at %016lx nip %016lx lr %016lx code %x\n"; if (!user_mode(regs)) { die("Exception in kernel mode", regs, signr); return; } - if (show_unhandled_signals && unhandled_signal(current, signr)) { - printk_ratelimited(regs->msr & MSR_64BIT ? fmt64 : fmt32, - current->comm, current->pid, signr, - addr, regs->nip, regs->link, code); - } + show_signal_msg(signr, regs, code, addr); if (arch_irqs_disabled() && !arch_irq_disabled_regs(regs)) local_irq_enable(); diff --git a/arch/powerpc/kernel/vdso.c b/arch/powerpc/kernel/vdso.c index d2205b97628c..65b3bdb99f0b 100644 --- a/arch/powerpc/kernel/vdso.c +++ b/arch/powerpc/kernel/vdso.c @@ -22,7 +22,6 @@ #include <linux/security.h> #include <linux/memblock.h> -#include <asm/cpu_has_feature.h> #include <asm/pgtable.h> #include <asm/processor.h> #include <asm/mmu.h> diff --git a/arch/powerpc/kernel/vector.S b/arch/powerpc/kernel/vector.S index f314fd475491..21165da0052d 100644 --- a/arch/powerpc/kernel/vector.S +++ b/arch/powerpc/kernel/vector.S @@ -8,6 +8,7 @@ #include <asm/page.h> #include <asm/ptrace.h> #include <asm/export.h> +#include <asm/asm-compat.h> /* * Load state from memory into VMX registers including VSCR. diff --git a/arch/powerpc/kernel/vmlinux.lds.S b/arch/powerpc/kernel/vmlinux.lds.S index 5baac79df97e..07ae018e550e 100644 --- a/arch/powerpc/kernel/vmlinux.lds.S +++ b/arch/powerpc/kernel/vmlinux.lds.S @@ -153,14 +153,16 @@ SECTIONS *(__rfi_flush_fixup) __stop___rfi_flush_fixup = .; } +#endif /* CONFIG_PPC64 */ +#ifdef CONFIG_PPC_BARRIER_NOSPEC . = ALIGN(8); __spec_barrier_fixup : AT(ADDR(__spec_barrier_fixup) - LOAD_OFFSET) { __start___barrier_nospec_fixup = .; *(__barrier_nospec_fixup) __stop___barrier_nospec_fixup = .; } -#endif +#endif /* CONFIG_PPC_BARRIER_NOSPEC */ EXCEPTION_TABLE(0) diff --git a/arch/powerpc/kernel/watchdog.c b/arch/powerpc/kernel/watchdog.c index 1d82274f7e9f..3c6ab22a0c4e 100644 --- a/arch/powerpc/kernel/watchdog.c +++ b/arch/powerpc/kernel/watchdog.c @@ -174,7 +174,6 @@ static void watchdog_smp_panic(int cpu, u64 tb) continue; smp_send_nmi_ipi(c, wd_lockup_ipi, 1000000); } - smp_flush_nmi_ipi(1000000); } /* Take the stuck CPUs out of the watch group */ diff --git a/arch/powerpc/kvm/book3s.c b/arch/powerpc/kvm/book3s.c index edaf4720d156..87348e498c89 100644 --- a/arch/powerpc/kvm/book3s.c +++ b/arch/powerpc/kvm/book3s.c @@ -28,7 +28,6 @@ #include <asm/reg.h> #include <asm/cputable.h> #include <asm/cacheflush.h> -#include <asm/tlbflush.h> #include <linux/uaccess.h> #include <asm/io.h> #include <asm/kvm_ppc.h> diff --git a/arch/powerpc/kvm/book3s_32_mmu.c b/arch/powerpc/kvm/book3s_32_mmu.c index 45c8ea4a0487..612169988a3d 100644 --- a/arch/powerpc/kvm/book3s_32_mmu.c +++ b/arch/powerpc/kvm/book3s_32_mmu.c @@ -23,7 +23,6 @@ #include <linux/kvm_host.h> #include <linux/highmem.h> -#include <asm/tlbflush.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> diff --git a/arch/powerpc/kvm/book3s_64_mmu.c b/arch/powerpc/kvm/book3s_64_mmu.c index cf9d686e8162..c92dd25bed23 100644 --- a/arch/powerpc/kvm/book3s_64_mmu.c +++ b/arch/powerpc/kvm/book3s_64_mmu.c @@ -23,7 +23,6 @@ #include <linux/kvm_host.h> #include <linux/highmem.h> -#include <asm/tlbflush.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> #include <asm/book3s/64/mmu-hash.h> diff --git a/arch/powerpc/kvm/book3s_64_mmu_hv.c b/arch/powerpc/kvm/book3s_64_mmu_hv.c index 7f3a8cf5d66f..3c0e8fb2b773 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_hv.c +++ b/arch/powerpc/kvm/book3s_64_mmu_hv.c @@ -29,7 +29,6 @@ #include <linux/file.h> #include <linux/debugfs.h> -#include <asm/tlbflush.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> #include <asm/book3s/64/mmu-hash.h> diff --git a/arch/powerpc/kvm/book3s_64_mmu_radix.c b/arch/powerpc/kvm/book3s_64_mmu_radix.c index 176f911ee983..0af1c0aea1fe 100644 --- a/arch/powerpc/kvm/book3s_64_mmu_radix.c +++ b/arch/powerpc/kvm/book3s_64_mmu_radix.c @@ -66,10 +66,7 @@ int kvmppc_mmu_radix_xlate(struct kvm_vcpu *vcpu, gva_t eaddr, bits = root & RPDS_MASK; root = root & RPDB_MASK; - /* P9 DD1 interprets RTS (radix tree size) differently */ offset = rts + 31; - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - offset -= 3; /* current implementations only support 52-bit space */ if (offset != 52) @@ -160,17 +157,7 @@ static unsigned long kvmppc_radix_update_pte(struct kvm *kvm, pte_t *ptep, unsigned long clr, unsigned long set, unsigned long addr, unsigned int shift) { - unsigned long old = 0; - - if (!(clr & _PAGE_PRESENT) && cpu_has_feature(CPU_FTR_POWER9_DD1) && - pte_present(*ptep)) { - /* have to invalidate it first */ - old = __radix_pte_update(ptep, _PAGE_PRESENT, 0); - kvmppc_radix_tlbie_page(kvm, addr, shift); - set |= _PAGE_PRESENT; - old &= _PAGE_PRESENT; - } - return __radix_pte_update(ptep, clr, set) | old; + return __radix_pte_update(ptep, clr, set); } void kvmppc_radix_set_pte_at(struct kvm *kvm, unsigned long addr, diff --git a/arch/powerpc/kvm/book3s_64_slb.S b/arch/powerpc/kvm/book3s_64_slb.S index 688722acd692..066c665dc86f 100644 --- a/arch/powerpc/kvm/book3s_64_slb.S +++ b/arch/powerpc/kvm/book3s_64_slb.S @@ -17,6 +17,9 @@ * Authors: Alexander Graf <agraf@suse.de> */ +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> + #define SHADOW_SLB_ENTRY_LEN 0x10 #define OFFSET_ESID(x) (SHADOW_SLB_ENTRY_LEN * x) #define OFFSET_VSID(x) ((SHADOW_SLB_ENTRY_LEN * x) + 8) diff --git a/arch/powerpc/kvm/book3s_64_vio.c b/arch/powerpc/kvm/book3s_64_vio.c index 8c456fa691a5..e8afdd4f4814 100644 --- a/arch/powerpc/kvm/book3s_64_vio.c +++ b/arch/powerpc/kvm/book3s_64_vio.c @@ -31,7 +31,6 @@ #include <linux/iommu.h> #include <linux/file.h> -#include <asm/tlbflush.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> #include <asm/book3s/64/mmu-hash.h> @@ -378,19 +377,19 @@ static long kvmppc_tce_iommu_mapped_dec(struct kvm *kvm, { struct mm_iommu_table_group_mem_t *mem = NULL; const unsigned long pgsize = 1ULL << tbl->it_page_shift; - unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); + __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); if (!pua) /* it_userspace allocation might be delayed */ return H_TOO_HARD; - mem = mm_iommu_lookup(kvm->mm, *pua, pgsize); + mem = mm_iommu_lookup(kvm->mm, be64_to_cpu(*pua), pgsize); if (!mem) return H_TOO_HARD; mm_iommu_mapped_dec(mem); - *pua = 0; + *pua = cpu_to_be64(0); return H_SUCCESS; } @@ -437,7 +436,8 @@ long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl, enum dma_data_direction dir) { long ret; - unsigned long hpa, *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); + unsigned long hpa; + __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); struct mm_iommu_table_group_mem_t *mem; if (!pua) @@ -464,7 +464,7 @@ long kvmppc_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl, if (dir != DMA_NONE) kvmppc_tce_iommu_mapped_dec(kvm, tbl, entry); - *pua = ua; + *pua = cpu_to_be64(ua); return 0; } diff --git a/arch/powerpc/kvm/book3s_64_vio_hv.c b/arch/powerpc/kvm/book3s_64_vio_hv.c index 5b298f5a1a14..506a4d400458 100644 --- a/arch/powerpc/kvm/book3s_64_vio_hv.c +++ b/arch/powerpc/kvm/book3s_64_vio_hv.c @@ -26,8 +26,8 @@ #include <linux/slab.h> #include <linux/hugetlb.h> #include <linux/list.h> +#include <linux/stringify.h> -#include <asm/tlbflush.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> #include <asm/book3s/64/mmu-hash.h> @@ -200,23 +200,19 @@ static long kvmppc_rm_tce_iommu_mapped_dec(struct kvm *kvm, { struct mm_iommu_table_group_mem_t *mem = NULL; const unsigned long pgsize = 1ULL << tbl->it_page_shift; - unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); + __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry); if (!pua) /* it_userspace allocation might be delayed */ return H_TOO_HARD; - pua = (void *) vmalloc_to_phys(pua); - if (WARN_ON_ONCE_RM(!pua)) - return H_HARDWARE; - - mem = mm_iommu_lookup_rm(kvm->mm, *pua, pgsize); + mem = mm_iommu_lookup_rm(kvm->mm, be64_to_cpu(*pua), pgsize); if (!mem) return H_TOO_HARD; mm_iommu_mapped_dec(mem); - *pua = 0; + *pua = cpu_to_be64(0); return H_SUCCESS; } @@ -268,7 +264,7 @@ static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl, { long ret; unsigned long hpa = 0; - unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); + __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY_RM(tbl, entry); struct mm_iommu_table_group_mem_t *mem; if (!pua) @@ -283,10 +279,6 @@ static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl, &hpa))) return H_HARDWARE; - pua = (void *) vmalloc_to_phys(pua); - if (WARN_ON_ONCE_RM(!pua)) - return H_HARDWARE; - if (WARN_ON_ONCE_RM(mm_iommu_mapped_inc(mem))) return H_CLOSED; @@ -303,7 +295,7 @@ static long kvmppc_rm_tce_iommu_do_map(struct kvm *kvm, struct iommu_table *tbl, if (dir != DMA_NONE) kvmppc_rm_tce_iommu_mapped_dec(kvm, tbl, entry); - *pua = ua; + *pua = cpu_to_be64(ua); return 0; } diff --git a/arch/powerpc/kvm/book3s_hv.c b/arch/powerpc/kvm/book3s_hv.c index ee4a8854985e..96f1cc6c97b7 100644 --- a/arch/powerpc/kvm/book3s_hv.c +++ b/arch/powerpc/kvm/book3s_hv.c @@ -53,7 +53,6 @@ #include <asm/disassemble.h> #include <asm/cputable.h> #include <asm/cacheflush.h> -#include <asm/tlbflush.h> #include <linux/uaccess.h> #include <asm/io.h> #include <asm/kvm_ppc.h> @@ -1693,14 +1692,6 @@ static int kvmppc_set_one_reg_hv(struct kvm_vcpu *vcpu, u64 id, r = set_vpa(vcpu, &vcpu->arch.dtl, addr, len); break; case KVM_REG_PPC_TB_OFFSET: - /* - * POWER9 DD1 has an erratum where writing TBU40 causes - * the timebase to lose ticks. So we don't let the - * timebase offset be changed on P9 DD1. (It is - * initialized to zero.) - */ - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - break; /* round up to multiple of 2^24 */ vcpu->arch.vcore->tb_offset = ALIGN(set_reg_val(id, *val), 1UL << 24); @@ -2026,8 +2017,6 @@ static struct kvm_vcpu *kvmppc_core_vcpu_create_hv(struct kvm *kvm, /* * Set the default HFSCR for the guest from the host value. * This value is only used on POWER9. - * On POWER9 DD1, TM doesn't work, so we make sure to - * prevent the guest from using it. * On POWER9, we want to virtualize the doorbell facility, so we * turn off the HFSCR bit, which causes those instructions to trap. */ diff --git a/arch/powerpc/kvm/book3s_hv_interrupts.S b/arch/powerpc/kvm/book3s_hv_interrupts.S index 82f2ff9410b6..666b91c79eb4 100644 --- a/arch/powerpc/kvm/book3s_hv_interrupts.S +++ b/arch/powerpc/kvm/book3s_hv_interrupts.S @@ -27,6 +27,8 @@ #include <asm/asm-offsets.h> #include <asm/exception-64s.h> #include <asm/ppc-opcode.h> +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> /***************************************************************************** * * diff --git a/arch/powerpc/kvm/book3s_hv_rm_mmu.c b/arch/powerpc/kvm/book3s_hv_rm_mmu.c index 1f22d9e977d4..a67cf1cdeda4 100644 --- a/arch/powerpc/kvm/book3s_hv_rm_mmu.c +++ b/arch/powerpc/kvm/book3s_hv_rm_mmu.c @@ -14,7 +14,6 @@ #include <linux/module.h> #include <linux/log2.h> -#include <asm/tlbflush.h> #include <asm/trace.h> #include <asm/kvm_ppc.h> #include <asm/kvm_book3s.h> diff --git a/arch/powerpc/kvm/book3s_hv_rmhandlers.S b/arch/powerpc/kvm/book3s_hv_rmhandlers.S index 153988d878e8..1d14046124a0 100644 --- a/arch/powerpc/kvm/book3s_hv_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_hv_rmhandlers.S @@ -32,6 +32,8 @@ #include <asm/opal.h> #include <asm/xive-regs.h> #include <asm/thread_info.h> +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> /* Sign-extend HDEC if not on POWER9 */ #define EXTEND_HDEC(reg) \ @@ -917,9 +919,6 @@ END_FTR_SECTION_IFSET(CPU_FTR_DAWR) mtspr SPRN_PID, r7 mtspr SPRN_WORT, r8 BEGIN_FTR_SECTION - PPC_INVALIDATE_ERAT -END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) -BEGIN_FTR_SECTION /* POWER8-only registers */ ld r5, VCPU_TCSCR(r4) ld r6, VCPU_ACOP(r4) @@ -1912,7 +1911,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) ld r5, VCPU_KVM(r9) lbz r0, KVM_RADIX(r5) cmpwi cr2, r0, 0 - beq cr2, 4f + beq cr2, 2f /* * Radix: do eieio; tlbsync; ptesync sequence in case we @@ -1952,11 +1951,7 @@ END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300) bdnz 1b ptesync -2: /* Flush the ERAT on radix P9 DD1 guest exit */ -BEGIN_FTR_SECTION - PPC_INVALIDATE_ERAT -END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) -4: +2: #endif /* CONFIG_PPC_RADIX_MMU */ /* @@ -3367,11 +3362,6 @@ END_FTR_SECTION_IFCLR(CPU_FTR_ARCH_300) mtspr SPRN_CIABR, r0 mtspr SPRN_DAWRX, r0 - /* Flush the ERAT on radix P9 DD1 guest exit */ -BEGIN_FTR_SECTION - PPC_INVALIDATE_ERAT -END_FTR_SECTION_IFSET(CPU_FTR_POWER9_DD1) - BEGIN_MMU_FTR_SECTION b 4f END_MMU_FTR_SECTION_IFSET(MMU_FTR_TYPE_RADIX) diff --git a/arch/powerpc/kvm/book3s_interrupts.S b/arch/powerpc/kvm/book3s_interrupts.S index c18e845019ec..d71dab16dc6f 100644 --- a/arch/powerpc/kvm/book3s_interrupts.S +++ b/arch/powerpc/kvm/book3s_interrupts.S @@ -23,6 +23,7 @@ #include <asm/page.h> #include <asm/asm-offsets.h> #include <asm/exception-64s.h> +#include <asm/asm-compat.h> #if defined(CONFIG_PPC_BOOK3S_64) #ifdef PPC64_ELF_ABI_v2 diff --git a/arch/powerpc/kvm/book3s_pr.c b/arch/powerpc/kvm/book3s_pr.c index c3b8006f0eac..47ee43bbd696 100644 --- a/arch/powerpc/kvm/book3s_pr.c +++ b/arch/powerpc/kvm/book3s_pr.c @@ -27,7 +27,6 @@ #include <asm/reg.h> #include <asm/cputable.h> #include <asm/cacheflush.h> -#include <asm/tlbflush.h> #include <linux/uaccess.h> #include <asm/io.h> #include <asm/kvm_ppc.h> diff --git a/arch/powerpc/kvm/book3s_rmhandlers.S b/arch/powerpc/kvm/book3s_rmhandlers.S index 34a5adeff084..b0089e04c8c8 100644 --- a/arch/powerpc/kvm/book3s_rmhandlers.S +++ b/arch/powerpc/kvm/book3s_rmhandlers.S @@ -23,6 +23,7 @@ #include <asm/mmu.h> #include <asm/page.h> #include <asm/asm-offsets.h> +#include <asm/asm-compat.h> #ifdef CONFIG_PPC_BOOK3S_64 #include <asm/exception-64s.h> diff --git a/arch/powerpc/kvm/book3s_segment.S b/arch/powerpc/kvm/book3s_segment.S index 98ccc7ec5d48..e5c542a7c5ac 100644 --- a/arch/powerpc/kvm/book3s_segment.S +++ b/arch/powerpc/kvm/book3s_segment.S @@ -19,6 +19,9 @@ /* Real mode helpers */ +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> + #if defined(CONFIG_PPC_BOOK3S_64) #define GET_SHADOW_VCPU(reg) \ diff --git a/arch/powerpc/kvm/book3s_xive_template.c b/arch/powerpc/kvm/book3s_xive_template.c index 6e41ba7ec8f4..4171ede8722b 100644 --- a/arch/powerpc/kvm/book3s_xive_template.c +++ b/arch/powerpc/kvm/book3s_xive_template.c @@ -25,18 +25,6 @@ static void GLUE(X_PFX,ack_pending)(struct kvmppc_xive_vcpu *xc) */ eieio(); - /* - * DD1 bug workaround: If PIPR is less favored than CPPR - * ignore the interrupt or we might incorrectly lose an IPB - * bit. - */ - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { - __be64 qw1 = __x_readq(__x_tima + TM_QW1_OS); - u8 pipr = be64_to_cpu(qw1) & 0xff; - if (pipr >= xc->hw_cppr) - return; - } - /* Perform the acknowledge OS to register cycle. */ ack = be16_to_cpu(__x_readw(__x_tima + TM_SPC_ACK_OS_REG)); @@ -89,8 +77,15 @@ static void GLUE(X_PFX,source_eoi)(u32 hw_irq, struct xive_irq_data *xd) /* If the XIVE supports the new "store EOI facility, use it */ if (xd->flags & XIVE_IRQ_FLAG_STORE_EOI) __x_writeq(0, __x_eoi_page(xd) + XIVE_ESB_STORE_EOI); - else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW) { + else if (hw_irq && xd->flags & XIVE_IRQ_FLAG_EOI_FW) opal_int_eoi(hw_irq); + else if (xd->flags & XIVE_IRQ_FLAG_LSI) { + /* + * For LSIs the HW EOI cycle is used rather than PQ bits, + * as they are automatically re-triggred in HW when still + * pending. + */ + __x_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI); } else { uint64_t eoi_val; @@ -102,20 +97,12 @@ static void GLUE(X_PFX,source_eoi)(u32 hw_irq, struct xive_irq_data *xd) * * This allows us to then do a re-trigger if Q was set * rather than synthetizing an interrupt in software - * - * For LSIs, using the HW EOI cycle works around a problem - * on P9 DD1 PHBs where the other ESB accesses don't work - * properly. */ - if (xd->flags & XIVE_IRQ_FLAG_LSI) - __x_readq(__x_eoi_page(xd) + XIVE_ESB_LOAD_EOI); - else { - eoi_val = GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_00); - - /* Re-trigger if needed */ - if ((eoi_val & 1) && __x_trig_page(xd)) - __x_writeq(0, __x_trig_page(xd)); - } + eoi_val = GLUE(X_PFX,esb_load)(xd, XIVE_ESB_SET_PQ_00); + + /* Re-trigger if needed */ + if ((eoi_val & 1) && __x_trig_page(xd)) + __x_writeq(0, __x_trig_page(xd)); } } diff --git a/arch/powerpc/kvm/e500.c b/arch/powerpc/kvm/e500.c index f9f6468f4171..afd3c255a427 100644 --- a/arch/powerpc/kvm/e500.c +++ b/arch/powerpc/kvm/e500.c @@ -21,7 +21,6 @@ #include <asm/reg.h> #include <asm/cputable.h> -#include <asm/tlbflush.h> #include <asm/kvm_ppc.h> #include "../mm/mmu_decl.h" diff --git a/arch/powerpc/kvm/e500mc.c b/arch/powerpc/kvm/e500mc.c index d0b6b5788afc..d31645491a93 100644 --- a/arch/powerpc/kvm/e500mc.c +++ b/arch/powerpc/kvm/e500mc.c @@ -21,7 +21,6 @@ #include <asm/reg.h> #include <asm/cputable.h> -#include <asm/tlbflush.h> #include <asm/kvm_ppc.h> #include <asm/dbell.h> diff --git a/arch/powerpc/kvm/powerpc.c b/arch/powerpc/kvm/powerpc.c index 0e8c20c5eaac..3ccc386b380d 100644 --- a/arch/powerpc/kvm/powerpc.c +++ b/arch/powerpc/kvm/powerpc.c @@ -33,7 +33,6 @@ #include <asm/cputable.h> #include <linux/uaccess.h> #include <asm/kvm_ppc.h> -#include <asm/tlbflush.h> #include <asm/cputhreads.h> #include <asm/irqflags.h> #include <asm/iommu.h> diff --git a/arch/powerpc/lib/Makefile b/arch/powerpc/lib/Makefile index d0ca13ad8231..670286808928 100644 --- a/arch/powerpc/lib/Makefile +++ b/arch/powerpc/lib/Makefile @@ -12,7 +12,7 @@ CFLAGS_REMOVE_feature-fixups.o = $(CC_FLAGS_FTRACE) obj-y += string.o alloc.o code-patching.o feature-fixups.o -obj-$(CONFIG_PPC32) += div64.o copy_32.o crtsavres.o +obj-$(CONFIG_PPC32) += div64.o copy_32.o crtsavres.o strlen_32.o # See corresponding test in arch/powerpc/Makefile # 64-bit linker creates .sfpr on demand for final link (vmlinux), diff --git a/arch/powerpc/lib/code-patching.c b/arch/powerpc/lib/code-patching.c index e0d881ab304e..850f3b8f4da5 100644 --- a/arch/powerpc/lib/code-patching.c +++ b/arch/powerpc/lib/code-patching.c @@ -195,6 +195,22 @@ int patch_branch(unsigned int *addr, unsigned long target, int flags) return patch_instruction(addr, create_branch(addr, target, flags)); } +int patch_branch_site(s32 *site, unsigned long target, int flags) +{ + unsigned int *addr; + + addr = (unsigned int *)((unsigned long)site + *site); + return patch_instruction(addr, create_branch(addr, target, flags)); +} + +int patch_instruction_site(s32 *site, unsigned int instr) +{ + unsigned int *addr; + + addr = (unsigned int *)((unsigned long)site + *site); + return patch_instruction(addr, instr); +} + bool is_offset_in_branch_range(long offset) { /* diff --git a/arch/powerpc/lib/copy_32.S b/arch/powerpc/lib/copy_32.S index da425bb6b369..ba66846fe973 100644 --- a/arch/powerpc/lib/copy_32.S +++ b/arch/powerpc/lib/copy_32.S @@ -13,6 +13,7 @@ #include <asm/errno.h> #include <asm/ppc_asm.h> #include <asm/export.h> +#include <asm/code-patching-asm.h> #define COPY_16_BYTES \ lwz r7,4(r4); \ @@ -107,8 +108,8 @@ _GLOBAL(memset) * Skip optimised bloc until cache is enabled. Will be replaced * by 'bne' during boot to use normal procedure if r4 is not zero */ -_GLOBAL(memset_nocache_branch) - b 2f +5: b 2f + patch_site 5b, patch__memset_nocache clrlwi r7,r6,32-LG_CACHELINE_BYTES add r8,r7,r5 @@ -168,7 +169,9 @@ _GLOBAL(memmove) /* fall through */ _GLOBAL(memcpy) - b generic_memcpy +1: b generic_memcpy + patch_site 1b, patch__memcpy_nocache + add r7,r3,r5 /* test if the src & dst overlap */ add r8,r4,r5 cmplw 0,r4,r7 diff --git a/arch/powerpc/lib/copypage_64.S b/arch/powerpc/lib/copypage_64.S index 8d5034f645f3..694390357667 100644 --- a/arch/powerpc/lib/copypage_64.S +++ b/arch/powerpc/lib/copypage_64.S @@ -11,6 +11,7 @@ #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/export.h> +#include <asm/feature-fixups.h> .section ".toc","aw" PPC64_CACHES: diff --git a/arch/powerpc/lib/copypage_power7.S b/arch/powerpc/lib/copypage_power7.S index 8fa73b7ab20e..e38f956f7d9f 100644 --- a/arch/powerpc/lib/copypage_power7.S +++ b/arch/powerpc/lib/copypage_power7.S @@ -57,7 +57,7 @@ _GLOBAL(copypage_power7) std r4,-STACKFRAMESIZE+STK_REG(R30)(r1) std r0,16(r1) stdu r1,-STACKFRAMESIZE(r1) - bl enter_vmx_copy + bl enter_vmx_ops cmpwi r3,0 ld r0,STACKFRAMESIZE+16(r1) ld r3,STK_REG(R31)(r1) @@ -100,7 +100,7 @@ _GLOBAL(copypage_power7) addi r3,r3,128 bdnz 1b - b exit_vmx_copy /* tail call optimise */ + b exit_vmx_ops /* tail call optimise */ #else li r0,(PAGE_SIZE/128) diff --git a/arch/powerpc/lib/copyuser_64.S b/arch/powerpc/lib/copyuser_64.S index 506677395681..96c514bee66b 100644 --- a/arch/powerpc/lib/copyuser_64.S +++ b/arch/powerpc/lib/copyuser_64.S @@ -9,6 +9,13 @@ #include <asm/processor.h> #include <asm/ppc_asm.h> #include <asm/export.h> +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> + +#ifndef SELFTEST_CASE +/* 0 == most CPUs, 1 == POWER6, 2 == Cell */ +#define SELFTEST_CASE 0 +#endif #ifdef __BIG_ENDIAN__ #define sLd sld /* Shift towards low-numbered address. */ @@ -18,6 +25,28 @@ #define sHd sld /* Shift towards high-numbered address. */ #endif +/* + * These macros are used to generate exception table entries. + * The exception handlers below use the original arguments + * (stored on the stack) and the point where we're up to in + * the destination buffer, i.e. the address of the first + * unmodified byte. Generally r3 points into the destination + * buffer, but the first unmodified byte is at a variable + * offset from r3. In the code below, the symbol r3_offset + * is set to indicate the current offset at each point in + * the code. This offset is then used as a negative offset + * from the exception handler code, and those instructions + * before the exception handlers are addi instructions that + * adjust r3 to point to the correct place. + */ + .macro lex /* exception handler for load */ +100: EX_TABLE(100b, .Lld_exc - r3_offset) + .endm + + .macro stex /* exception handler for store */ +100: EX_TABLE(100b, .Lst_exc - r3_offset) + .endm + .align 7 _GLOBAL_TOC(__copy_tofrom_user) #ifdef CONFIG_PPC_BOOK3S_64 @@ -28,7 +57,7 @@ FTR_SECTION_ELSE ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY) #endif _GLOBAL(__copy_tofrom_user_base) - /* first check for a whole page copy on a page boundary */ + /* first check for a 4kB copy on a 4kB boundary */ cmpldi cr1,r5,16 cmpdi cr6,r5,4096 or r0,r3,r4 @@ -49,6 +78,7 @@ _GLOBAL(__copy_tofrom_user_base) * At the time of writing the only CPU that has this combination of bits * set is Power6. */ +test_feature = (SELFTEST_CASE == 1) BEGIN_FTR_SECTION nop FTR_SECTION_ELSE @@ -57,6 +87,8 @@ ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \ CPU_FTR_UNALIGNED_LD_STD) .Ldst_aligned: addi r3,r3,-16 +r3_offset = 16 +test_feature = (SELFTEST_CASE == 0) BEGIN_FTR_SECTION andi. r0,r4,7 bne .Lsrc_unaligned @@ -64,57 +96,69 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) blt cr1,.Ldo_tail /* if < 16 bytes to copy */ srdi r0,r5,5 cmpdi cr1,r0,0 -20: ld r7,0(r4) -220: ld r6,8(r4) +lex; ld r7,0(r4) +lex; ld r6,8(r4) addi r4,r4,16 mtctr r0 andi. r0,r5,0x10 beq 22f addi r3,r3,16 +r3_offset = 0 addi r4,r4,-16 mr r9,r7 mr r8,r6 beq cr1,72f -21: ld r7,16(r4) -221: ld r6,24(r4) +21: +lex; ld r7,16(r4) +lex; ld r6,24(r4) addi r4,r4,32 -70: std r9,0(r3) -270: std r8,8(r3) -22: ld r9,0(r4) -222: ld r8,8(r4) -71: std r7,16(r3) -271: std r6,24(r3) +stex; std r9,0(r3) +r3_offset = 8 +stex; std r8,8(r3) +r3_offset = 16 +22: +lex; ld r9,0(r4) +lex; ld r8,8(r4) +stex; std r7,16(r3) +r3_offset = 24 +stex; std r6,24(r3) addi r3,r3,32 +r3_offset = 0 bdnz 21b -72: std r9,0(r3) -272: std r8,8(r3) +72: +stex; std r9,0(r3) +r3_offset = 8 +stex; std r8,8(r3) +r3_offset = 16 andi. r5,r5,0xf beq+ 3f addi r4,r4,16 .Ldo_tail: addi r3,r3,16 +r3_offset = 0 bf cr7*4+0,246f -244: ld r9,0(r4) +lex; ld r9,0(r4) addi r4,r4,8 -245: std r9,0(r3) +stex; std r9,0(r3) addi r3,r3,8 246: bf cr7*4+1,1f -23: lwz r9,0(r4) +lex; lwz r9,0(r4) addi r4,r4,4 -73: stw r9,0(r3) +stex; stw r9,0(r3) addi r3,r3,4 1: bf cr7*4+2,2f -44: lhz r9,0(r4) +lex; lhz r9,0(r4) addi r4,r4,2 -74: sth r9,0(r3) +stex; sth r9,0(r3) addi r3,r3,2 2: bf cr7*4+3,3f -45: lbz r9,0(r4) -75: stb r9,0(r3) +lex; lbz r9,0(r4) +stex; stb r9,0(r3) 3: li r3,0 blr .Lsrc_unaligned: +r3_offset = 16 srdi r6,r5,3 addi r5,r5,-16 subf r4,r0,r4 @@ -127,58 +171,69 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) add r5,r5,r0 bt cr7*4+0,28f -24: ld r9,0(r4) /* 3+2n loads, 2+2n stores */ -25: ld r0,8(r4) +lex; ld r9,0(r4) /* 3+2n loads, 2+2n stores */ +lex; ld r0,8(r4) sLd r6,r9,r10 -26: ldu r9,16(r4) +lex; ldu r9,16(r4) sHd r7,r0,r11 sLd r8,r0,r10 or r7,r7,r6 blt cr6,79f -27: ld r0,8(r4) +lex; ld r0,8(r4) b 2f -28: ld r0,0(r4) /* 4+2n loads, 3+2n stores */ -29: ldu r9,8(r4) +28: +lex; ld r0,0(r4) /* 4+2n loads, 3+2n stores */ +lex; ldu r9,8(r4) sLd r8,r0,r10 addi r3,r3,-8 +r3_offset = 24 blt cr6,5f -30: ld r0,8(r4) +lex; ld r0,8(r4) sHd r12,r9,r11 sLd r6,r9,r10 -31: ldu r9,16(r4) +lex; ldu r9,16(r4) or r12,r8,r12 sHd r7,r0,r11 sLd r8,r0,r10 addi r3,r3,16 +r3_offset = 8 beq cr6,78f 1: or r7,r7,r6 -32: ld r0,8(r4) -76: std r12,8(r3) +lex; ld r0,8(r4) +stex; std r12,8(r3) +r3_offset = 16 2: sHd r12,r9,r11 sLd r6,r9,r10 -33: ldu r9,16(r4) +lex; ldu r9,16(r4) or r12,r8,r12 -77: stdu r7,16(r3) +stex; stdu r7,16(r3) +r3_offset = 8 sHd r7,r0,r11 sLd r8,r0,r10 bdnz 1b -78: std r12,8(r3) +78: +stex; std r12,8(r3) +r3_offset = 16 or r7,r7,r6 -79: std r7,16(r3) +79: +stex; std r7,16(r3) +r3_offset = 24 5: sHd r12,r9,r11 or r12,r8,r12 -80: std r12,24(r3) +stex; std r12,24(r3) +r3_offset = 32 bne 6f li r3,0 blr 6: cmpwi cr1,r5,8 addi r3,r3,32 +r3_offset = 0 sLd r9,r9,r10 ble cr1,7f -34: ld r0,8(r4) +lex; ld r0,8(r4) sHd r7,r0,r11 or r9,r7,r9 7: @@ -186,7 +241,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) #ifdef __BIG_ENDIAN__ rotldi r9,r9,32 #endif -94: stw r9,0(r3) +stex; stw r9,0(r3) #ifdef __LITTLE_ENDIAN__ rotrdi r9,r9,32 #endif @@ -195,7 +250,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) #ifdef __BIG_ENDIAN__ rotldi r9,r9,16 #endif -95: sth r9,0(r3) +stex; sth r9,0(r3) #ifdef __LITTLE_ENDIAN__ rotrdi r9,r9,16 #endif @@ -204,7 +259,7 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) #ifdef __BIG_ENDIAN__ rotldi r9,r9,8 #endif -96: stb r9,0(r3) +stex; stb r9,0(r3) #ifdef __LITTLE_ENDIAN__ rotrdi r9,r9,8 #endif @@ -212,47 +267,55 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) blr .Ldst_unaligned: +r3_offset = 0 PPC_MTOCRF(0x01,r6) /* put #bytes to 8B bdry into cr7 */ subf r5,r6,r5 li r7,0 cmpldi cr1,r5,16 bf cr7*4+3,1f -35: lbz r0,0(r4) -81: stb r0,0(r3) +100: EX_TABLE(100b, .Lld_exc_r7) + lbz r0,0(r4) +100: EX_TABLE(100b, .Lst_exc_r7) + stb r0,0(r3) addi r7,r7,1 1: bf cr7*4+2,2f -36: lhzx r0,r7,r4 -82: sthx r0,r7,r3 +100: EX_TABLE(100b, .Lld_exc_r7) + lhzx r0,r7,r4 +100: EX_TABLE(100b, .Lst_exc_r7) + sthx r0,r7,r3 addi r7,r7,2 2: bf cr7*4+1,3f -37: lwzx r0,r7,r4 -83: stwx r0,r7,r3 +100: EX_TABLE(100b, .Lld_exc_r7) + lwzx r0,r7,r4 +100: EX_TABLE(100b, .Lst_exc_r7) + stwx r0,r7,r3 3: PPC_MTOCRF(0x01,r5) add r4,r6,r4 add r3,r6,r3 b .Ldst_aligned .Lshort_copy: +r3_offset = 0 bf cr7*4+0,1f -38: lwz r0,0(r4) -39: lwz r9,4(r4) +lex; lwz r0,0(r4) +lex; lwz r9,4(r4) addi r4,r4,8 -84: stw r0,0(r3) -85: stw r9,4(r3) +stex; stw r0,0(r3) +stex; stw r9,4(r3) addi r3,r3,8 1: bf cr7*4+1,2f -40: lwz r0,0(r4) +lex; lwz r0,0(r4) addi r4,r4,4 -86: stw r0,0(r3) +stex; stw r0,0(r3) addi r3,r3,4 2: bf cr7*4+2,3f -41: lhz r0,0(r4) +lex; lhz r0,0(r4) addi r4,r4,2 -87: sth r0,0(r3) +stex; sth r0,0(r3) addi r3,r3,2 3: bf cr7*4+3,4f -42: lbz r0,0(r4) -88: stb r0,0(r3) +lex; lbz r0,0(r4) +stex; stb r0,0(r3) 4: li r3,0 blr @@ -260,48 +323,34 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) * exception handlers follow * we have to return the number of bytes not copied * for an exception on a load, we set the rest of the destination to 0 + * Note that the number of bytes of instructions for adjusting r3 needs + * to equal the amount of the adjustment, due to the trick of using + * .Lld_exc - r3_offset as the handler address. */ -136: -137: +.Lld_exc_r7: add r3,r3,r7 - b 1f -130: -131: + b .Lld_exc + + /* adjust by 24 */ addi r3,r3,8 -120: -320: -122: -322: -124: -125: -126: -127: -128: -129: -133: + nop + /* adjust by 16 */ addi r3,r3,8 -132: + nop + /* adjust by 8 */ addi r3,r3,8 -121: -321: -344: -134: -135: -138: -139: -140: -141: -142: -123: -144: -145: + nop /* - * here we have had a fault on a load and r3 points to the first - * unmodified byte of the destination + * Here we have had a fault on a load and r3 points to the first + * unmodified byte of the destination. We use the original arguments + * and r3 to work out how much wasn't copied. Since we load some + * distance ahead of the stores, we continue copying byte-by-byte until + * we hit the load fault again in order to copy as much as possible. */ -1: ld r6,-24(r1) +.Lld_exc: + ld r6,-24(r1) ld r4,-16(r1) ld r5,-8(r1) subf r6,r6,r3 @@ -312,9 +361,11 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) * first see if we can copy any more bytes before hitting another exception */ mtctr r5 +r3_offset = 0 +100: EX_TABLE(100b, .Ldone) 43: lbz r0,0(r4) addi r4,r4,1 -89: stb r0,0(r3) +stex; stb r0,0(r3) addi r3,r3,1 bdnz 43b li r3,0 /* huh? all copied successfully this time? */ @@ -323,116 +374,63 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) /* * here we have trapped again, amount remaining is in ctr. */ -143: mfctr r3 +.Ldone: + mfctr r3 blr /* - * exception handlers for stores: we just need to work - * out how many bytes weren't copied + * exception handlers for stores: we need to work out how many bytes + * weren't copied, and we may need to copy some more. + * Note that the number of bytes of instructions for adjusting r3 needs + * to equal the amount of the adjustment, due to the trick of using + * .Lst_exc - r3_offset as the handler address. */ -182: -183: +.Lst_exc_r7: add r3,r3,r7 - b 1f -371: -180: + b .Lst_exc + + /* adjust by 24 */ addi r3,r3,8 -171: -177: -179: + nop + /* adjust by 16 */ addi r3,r3,8 -370: -372: -176: -178: + nop + /* adjust by 8 */ addi r3,r3,4 -185: + /* adjust by 4 */ addi r3,r3,4 -170: -172: -345: -173: -174: -175: -181: -184: -186: -187: -188: -189: -194: -195: -196: -1: - ld r6,-24(r1) - ld r5,-8(r1) - add r6,r6,r5 - subf r3,r3,r6 /* #bytes not copied */ +.Lst_exc: + ld r6,-24(r1) /* original destination pointer */ + ld r4,-16(r1) /* original source pointer */ + ld r5,-8(r1) /* original number of bytes */ + add r7,r6,r5 + /* + * If the destination pointer isn't 8-byte aligned, + * we may have got the exception as a result of a + * store that overlapped a page boundary, so we may be + * able to copy a few more bytes. + */ +17: andi. r0,r3,7 + beq 19f + subf r8,r6,r3 /* #bytes copied */ +100: EX_TABLE(100b,19f) + lbzx r0,r8,r4 +100: EX_TABLE(100b,19f) + stb r0,0(r3) + addi r3,r3,1 + cmpld r3,r7 + blt 17b +19: subf r3,r3,r7 /* #bytes not copied in r3 */ blr - EX_TABLE(20b,120b) - EX_TABLE(220b,320b) - EX_TABLE(21b,121b) - EX_TABLE(221b,321b) - EX_TABLE(70b,170b) - EX_TABLE(270b,370b) - EX_TABLE(22b,122b) - EX_TABLE(222b,322b) - EX_TABLE(71b,171b) - EX_TABLE(271b,371b) - EX_TABLE(72b,172b) - EX_TABLE(272b,372b) - EX_TABLE(244b,344b) - EX_TABLE(245b,345b) - EX_TABLE(23b,123b) - EX_TABLE(73b,173b) - EX_TABLE(44b,144b) - EX_TABLE(74b,174b) - EX_TABLE(45b,145b) - EX_TABLE(75b,175b) - EX_TABLE(24b,124b) - EX_TABLE(25b,125b) - EX_TABLE(26b,126b) - EX_TABLE(27b,127b) - EX_TABLE(28b,128b) - EX_TABLE(29b,129b) - EX_TABLE(30b,130b) - EX_TABLE(31b,131b) - EX_TABLE(32b,132b) - EX_TABLE(76b,176b) - EX_TABLE(33b,133b) - EX_TABLE(77b,177b) - EX_TABLE(78b,178b) - EX_TABLE(79b,179b) - EX_TABLE(80b,180b) - EX_TABLE(34b,134b) - EX_TABLE(94b,194b) - EX_TABLE(95b,195b) - EX_TABLE(96b,196b) - EX_TABLE(35b,135b) - EX_TABLE(81b,181b) - EX_TABLE(36b,136b) - EX_TABLE(82b,182b) - EX_TABLE(37b,137b) - EX_TABLE(83b,183b) - EX_TABLE(38b,138b) - EX_TABLE(39b,139b) - EX_TABLE(84b,184b) - EX_TABLE(85b,185b) - EX_TABLE(40b,140b) - EX_TABLE(86b,186b) - EX_TABLE(41b,141b) - EX_TABLE(87b,187b) - EX_TABLE(42b,142b) - EX_TABLE(88b,188b) - EX_TABLE(43b,143b) - EX_TABLE(89b,189b) - /* * Routine to copy a whole page of data, optimized for POWER4. * On POWER4 it is more than 50% faster than the simple loop * above (following the .Ldst_aligned label). */ + .macro exc +100: EX_TABLE(100b, .Labort) + .endm .Lcopy_page_4K: std r31,-32(1) std r30,-40(1) @@ -451,86 +449,86 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) li r0,5 0: addi r5,r5,-24 mtctr r0 -20: ld r22,640(4) -21: ld r21,512(4) -22: ld r20,384(4) -23: ld r11,256(4) -24: ld r9,128(4) -25: ld r7,0(4) -26: ld r25,648(4) -27: ld r24,520(4) -28: ld r23,392(4) -29: ld r10,264(4) -30: ld r8,136(4) -31: ldu r6,8(4) +exc; ld r22,640(4) +exc; ld r21,512(4) +exc; ld r20,384(4) +exc; ld r11,256(4) +exc; ld r9,128(4) +exc; ld r7,0(4) +exc; ld r25,648(4) +exc; ld r24,520(4) +exc; ld r23,392(4) +exc; ld r10,264(4) +exc; ld r8,136(4) +exc; ldu r6,8(4) cmpwi r5,24 1: -32: std r22,648(3) -33: std r21,520(3) -34: std r20,392(3) -35: std r11,264(3) -36: std r9,136(3) -37: std r7,8(3) -38: ld r28,648(4) -39: ld r27,520(4) -40: ld r26,392(4) -41: ld r31,264(4) -42: ld r30,136(4) -43: ld r29,8(4) -44: std r25,656(3) -45: std r24,528(3) -46: std r23,400(3) -47: std r10,272(3) -48: std r8,144(3) -49: std r6,16(3) -50: ld r22,656(4) -51: ld r21,528(4) -52: ld r20,400(4) -53: ld r11,272(4) -54: ld r9,144(4) -55: ld r7,16(4) -56: std r28,664(3) -57: std r27,536(3) -58: std r26,408(3) -59: std r31,280(3) -60: std r30,152(3) -61: stdu r29,24(3) -62: ld r25,664(4) -63: ld r24,536(4) -64: ld r23,408(4) -65: ld r10,280(4) -66: ld r8,152(4) -67: ldu r6,24(4) +exc; std r22,648(3) +exc; std r21,520(3) +exc; std r20,392(3) +exc; std r11,264(3) +exc; std r9,136(3) +exc; std r7,8(3) +exc; ld r28,648(4) +exc; ld r27,520(4) +exc; ld r26,392(4) +exc; ld r31,264(4) +exc; ld r30,136(4) +exc; ld r29,8(4) +exc; std r25,656(3) +exc; std r24,528(3) +exc; std r23,400(3) +exc; std r10,272(3) +exc; std r8,144(3) +exc; std r6,16(3) +exc; ld r22,656(4) +exc; ld r21,528(4) +exc; ld r20,400(4) +exc; ld r11,272(4) +exc; ld r9,144(4) +exc; ld r7,16(4) +exc; std r28,664(3) +exc; std r27,536(3) +exc; std r26,408(3) +exc; std r31,280(3) +exc; std r30,152(3) +exc; stdu r29,24(3) +exc; ld r25,664(4) +exc; ld r24,536(4) +exc; ld r23,408(4) +exc; ld r10,280(4) +exc; ld r8,152(4) +exc; ldu r6,24(4) bdnz 1b -68: std r22,648(3) -69: std r21,520(3) -70: std r20,392(3) -71: std r11,264(3) -72: std r9,136(3) -73: std r7,8(3) -74: addi r4,r4,640 -75: addi r3,r3,648 +exc; std r22,648(3) +exc; std r21,520(3) +exc; std r20,392(3) +exc; std r11,264(3) +exc; std r9,136(3) +exc; std r7,8(3) + addi r4,r4,640 + addi r3,r3,648 bge 0b mtctr r5 -76: ld r7,0(4) -77: ld r8,8(4) -78: ldu r9,16(4) +exc; ld r7,0(4) +exc; ld r8,8(4) +exc; ldu r9,16(4) 3: -79: ld r10,8(4) -80: std r7,8(3) -81: ld r7,16(4) -82: std r8,16(3) -83: ld r8,24(4) -84: std r9,24(3) -85: ldu r9,32(4) -86: stdu r10,32(3) +exc; ld r10,8(4) +exc; std r7,8(3) +exc; ld r7,16(4) +exc; std r8,16(3) +exc; ld r8,24(4) +exc; std r9,24(3) +exc; ldu r9,32(4) +exc; stdu r10,32(3) bdnz 3b 4: -87: ld r10,8(4) -88: std r7,8(3) -89: std r8,16(3) -90: std r9,24(3) -91: std r10,32(3) +exc; ld r10,8(4) +exc; std r7,8(3) +exc; std r8,16(3) +exc; std r9,24(3) +exc; std r10,32(3) 9: ld r20,-120(1) ld r21,-112(1) ld r22,-104(1) @@ -550,7 +548,8 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) * on an exception, reset to the beginning and jump back into the * standard __copy_tofrom_user */ -100: ld r20,-120(1) +.Labort: + ld r20,-120(1) ld r21,-112(1) ld r22,-104(1) ld r23,-96(1) @@ -566,78 +565,4 @@ END_FTR_SECTION_IFCLR(CPU_FTR_UNALIGNED_LD_STD) ld r4,-16(r1) li r5,4096 b .Ldst_aligned - - EX_TABLE(20b,100b) - EX_TABLE(21b,100b) - EX_TABLE(22b,100b) - EX_TABLE(23b,100b) - EX_TABLE(24b,100b) - EX_TABLE(25b,100b) - EX_TABLE(26b,100b) - EX_TABLE(27b,100b) - EX_TABLE(28b,100b) - EX_TABLE(29b,100b) - EX_TABLE(30b,100b) - EX_TABLE(31b,100b) - EX_TABLE(32b,100b) - EX_TABLE(33b,100b) - EX_TABLE(34b,100b) - EX_TABLE(35b,100b) - EX_TABLE(36b,100b) - EX_TABLE(37b,100b) - EX_TABLE(38b,100b) - EX_TABLE(39b,100b) - EX_TABLE(40b,100b) - EX_TABLE(41b,100b) - EX_TABLE(42b,100b) - EX_TABLE(43b,100b) - EX_TABLE(44b,100b) - EX_TABLE(45b,100b) - EX_TABLE(46b,100b) - EX_TABLE(47b,100b) - EX_TABLE(48b,100b) - EX_TABLE(49b,100b) - EX_TABLE(50b,100b) - EX_TABLE(51b,100b) - EX_TABLE(52b,100b) - EX_TABLE(53b,100b) - EX_TABLE(54b,100b) - EX_TABLE(55b,100b) - EX_TABLE(56b,100b) - EX_TABLE(57b,100b) - EX_TABLE(58b,100b) - EX_TABLE(59b,100b) - EX_TABLE(60b,100b) - EX_TABLE(61b,100b) - EX_TABLE(62b,100b) - EX_TABLE(63b,100b) - EX_TABLE(64b,100b) - EX_TABLE(65b,100b) - EX_TABLE(66b,100b) - EX_TABLE(67b,100b) - EX_TABLE(68b,100b) - EX_TABLE(69b,100b) - EX_TABLE(70b,100b) - EX_TABLE(71b,100b) - EX_TABLE(72b,100b) - EX_TABLE(73b,100b) - EX_TABLE(74b,100b) - EX_TABLE(75b,100b) - EX_TABLE(76b,100b) - EX_TABLE(77b,100b) - EX_TABLE(78b,100b) - EX_TABLE(79b,100b) - EX_TABLE(80b,100b) - EX_TABLE(81b,100b) - EX_TABLE(82b,100b) - EX_TABLE(83b,100b) - EX_TABLE(84b,100b) - EX_TABLE(85b,100b) - EX_TABLE(86b,100b) - EX_TABLE(87b,100b) - EX_TABLE(88b,100b) - EX_TABLE(89b,100b) - EX_TABLE(90b,100b) - EX_TABLE(91b,100b) - EXPORT_SYMBOL(__copy_tofrom_user) diff --git a/arch/powerpc/lib/copyuser_power7.S b/arch/powerpc/lib/copyuser_power7.S index 215e4760c09f..1a1fe180af62 100644 --- a/arch/powerpc/lib/copyuser_power7.S +++ b/arch/powerpc/lib/copyuser_power7.S @@ -19,6 +19,11 @@ */ #include <asm/ppc_asm.h> +#ifndef SELFTEST_CASE +/* 0 == don't use VMX, 1 == use VMX */ +#define SELFTEST_CASE 0 +#endif + #ifdef __BIG_ENDIAN__ #define LVS(VRT,RA,RB) lvsl VRT,RA,RB #define VPERM(VRT,VRA,VRB,VRC) vperm VRT,VRA,VRB,VRC @@ -80,7 +85,6 @@ _GLOBAL(__copy_tofrom_user_power7) -#ifdef CONFIG_ALTIVEC cmpldi r5,16 cmpldi cr1,r5,3328 @@ -89,15 +93,12 @@ _GLOBAL(__copy_tofrom_user_power7) std r5,-STACKFRAMESIZE+STK_REG(R29)(r1) blt .Lshort_copy - bge cr1,.Lvmx_copy -#else - cmpldi r5,16 - std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) - std r4,-STACKFRAMESIZE+STK_REG(R30)(r1) - std r5,-STACKFRAMESIZE+STK_REG(R29)(r1) - - blt .Lshort_copy +#ifdef CONFIG_ALTIVEC +test_feature = SELFTEST_CASE +BEGIN_FTR_SECTION + bgt cr1,.Lvmx_copy +END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) #endif .Lnonvmx_copy: @@ -278,8 +279,8 @@ err1; stb r0,0(r3) addi r1,r1,STACKFRAMESIZE b .Lnonvmx_copy -#ifdef CONFIG_ALTIVEC .Lvmx_copy: +#ifdef CONFIG_ALTIVEC mflr r0 std r0,16(r1) stdu r1,-STACKFRAMESIZE(r1) diff --git a/arch/powerpc/lib/feature-fixups-test.S b/arch/powerpc/lib/feature-fixups-test.S index f16cec989506..ee7c5fd5fc64 100644 --- a/arch/powerpc/lib/feature-fixups-test.S +++ b/arch/powerpc/lib/feature-fixups-test.S @@ -11,6 +11,7 @@ #include <asm/feature-fixups.h> #include <asm/ppc_asm.h> #include <asm/synch.h> +#include <asm/asm-compat.h> .text diff --git a/arch/powerpc/lib/feature-fixups.c b/arch/powerpc/lib/feature-fixups.c index 8b69f868298c..e613b02bb2f0 100644 --- a/arch/powerpc/lib/feature-fixups.c +++ b/arch/powerpc/lib/feature-fixups.c @@ -304,6 +304,9 @@ void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_ printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i); } +#endif /* CONFIG_PPC_BOOK3S_64 */ + +#ifdef CONFIG_PPC_BARRIER_NOSPEC void do_barrier_nospec_fixups(bool enable) { void *start, *end; @@ -313,8 +316,38 @@ void do_barrier_nospec_fixups(bool enable) do_barrier_nospec_fixups_range(enable, start, end); } +#endif /* CONFIG_PPC_BARRIER_NOSPEC */ -#endif /* CONFIG_PPC_BOOK3S_64 */ +#ifdef CONFIG_PPC_FSL_BOOK3E +void do_barrier_nospec_fixups_range(bool enable, void *fixup_start, void *fixup_end) +{ + unsigned int instr[2], *dest; + long *start, *end; + int i; + + start = fixup_start; + end = fixup_end; + + instr[0] = PPC_INST_NOP; + instr[1] = PPC_INST_NOP; + + if (enable) { + pr_info("barrier-nospec: using isync; sync as speculation barrier\n"); + instr[0] = PPC_INST_ISYNC; + instr[1] = PPC_INST_SYNC; + } + + for (i = 0; start < end; start++, i++) { + dest = (void *)start + *start; + + pr_devel("patching dest %lx\n", (unsigned long)dest); + patch_instruction(dest, instr[0]); + patch_instruction(dest + 1, instr[1]); + } + + printk(KERN_DEBUG "barrier-nospec: patched %d locations\n", i); +} +#endif /* CONFIG_PPC_FSL_BOOK3E */ void do_lwsync_fixups(unsigned long value, void *fixup_start, void *fixup_end) { diff --git a/arch/powerpc/lib/hweight_64.S b/arch/powerpc/lib/hweight_64.S index 3de7ac154f24..0526b2225260 100644 --- a/arch/powerpc/lib/hweight_64.S +++ b/arch/powerpc/lib/hweight_64.S @@ -20,6 +20,7 @@ #include <asm/processor.h> #include <asm/ppc_asm.h> #include <asm/export.h> +#include <asm/feature-fixups.h> /* Note: This code relies on -mminimal-toc */ diff --git a/arch/powerpc/lib/ldstfp.S b/arch/powerpc/lib/ldstfp.S index ae15eba49c1f..32e91994b6b2 100644 --- a/arch/powerpc/lib/ldstfp.S +++ b/arch/powerpc/lib/ldstfp.S @@ -15,6 +15,7 @@ #include <asm/ppc-opcode.h> #include <asm/reg.h> #include <asm/asm-offsets.h> +#include <asm/asm-compat.h> #include <linux/errno.h> #ifdef CONFIG_PPC_FPU diff --git a/arch/powerpc/lib/locks.c b/arch/powerpc/lib/locks.c index b7b1237d4aa6..35a0ef932e1a 100644 --- a/arch/powerpc/lib/locks.c +++ b/arch/powerpc/lib/locks.c @@ -15,7 +15,6 @@ #include <linux/kernel.h> #include <linux/spinlock.h> #include <linux/export.h> -#include <linux/stringify.h> #include <linux/smp.h> /* waiting for a spinlock... */ diff --git a/arch/powerpc/lib/memcmp_64.S b/arch/powerpc/lib/memcmp_64.S index d75d18b7bd55..844d8e774492 100644 --- a/arch/powerpc/lib/memcmp_64.S +++ b/arch/powerpc/lib/memcmp_64.S @@ -9,6 +9,7 @@ */ #include <asm/ppc_asm.h> #include <asm/export.h> +#include <asm/ppc-opcode.h> #define off8 r6 #define off16 r7 @@ -24,28 +25,102 @@ #define rH r31 #ifdef __LITTLE_ENDIAN__ +#define LH lhbrx +#define LW lwbrx #define LD ldbrx +#define LVS lvsr +#define VPERM(_VRT,_VRA,_VRB,_VRC) \ + vperm _VRT,_VRB,_VRA,_VRC #else +#define LH lhzx +#define LW lwzx #define LD ldx +#define LVS lvsl +#define VPERM(_VRT,_VRA,_VRB,_VRC) \ + vperm _VRT,_VRA,_VRB,_VRC #endif -_GLOBAL(memcmp) +#define VMX_THRESH 4096 +#define ENTER_VMX_OPS \ + mflr r0; \ + std r3,-STACKFRAMESIZE+STK_REG(R31)(r1); \ + std r4,-STACKFRAMESIZE+STK_REG(R30)(r1); \ + std r5,-STACKFRAMESIZE+STK_REG(R29)(r1); \ + std r0,16(r1); \ + stdu r1,-STACKFRAMESIZE(r1); \ + bl enter_vmx_ops; \ + cmpwi cr1,r3,0; \ + ld r0,STACKFRAMESIZE+16(r1); \ + ld r3,STK_REG(R31)(r1); \ + ld r4,STK_REG(R30)(r1); \ + ld r5,STK_REG(R29)(r1); \ + addi r1,r1,STACKFRAMESIZE; \ + mtlr r0 + +#define EXIT_VMX_OPS \ + mflr r0; \ + std r3,-STACKFRAMESIZE+STK_REG(R31)(r1); \ + std r4,-STACKFRAMESIZE+STK_REG(R30)(r1); \ + std r5,-STACKFRAMESIZE+STK_REG(R29)(r1); \ + std r0,16(r1); \ + stdu r1,-STACKFRAMESIZE(r1); \ + bl exit_vmx_ops; \ + ld r0,STACKFRAMESIZE+16(r1); \ + ld r3,STK_REG(R31)(r1); \ + ld r4,STK_REG(R30)(r1); \ + ld r5,STK_REG(R29)(r1); \ + addi r1,r1,STACKFRAMESIZE; \ + mtlr r0 + +/* + * LD_VSR_CROSS16B load the 2nd 16 bytes for _vaddr which is unaligned with + * 16 bytes boundary and permute the result with the 1st 16 bytes. + + * | y y y y y y y y y y y y y 0 1 2 | 3 4 5 6 7 8 9 a b c d e f z z z | + * ^ ^ ^ + * 0xbbbb10 0xbbbb20 0xbbb30 + * ^ + * _vaddr + * + * + * _vmask is the mask generated by LVS + * _v1st_qw is the 1st aligned QW of current addr which is already loaded. + * for example: 0xyyyyyyyyyyyyy012 for big endian + * _v2nd_qw is the 2nd aligned QW of cur _vaddr to be loaded. + * for example: 0x3456789abcdefzzz for big endian + * The permute result is saved in _v_res. + * for example: 0x0123456789abcdef for big endian. + */ +#define LD_VSR_CROSS16B(_vaddr,_vmask,_v1st_qw,_v2nd_qw,_v_res) \ + lvx _v2nd_qw,_vaddr,off16; \ + VPERM(_v_res,_v1st_qw,_v2nd_qw,_vmask) + +/* + * There are 2 categories for memcmp: + * 1) src/dst has the same offset to the 8 bytes boundary. The handlers + * are named like .Lsameoffset_xxxx + * 2) src/dst has different offset to the 8 bytes boundary. The handlers + * are named like .Ldiffoffset_xxxx + */ +_GLOBAL_TOC(memcmp) cmpdi cr1,r5,0 - /* Use the short loop if both strings are not 8B aligned */ - or r6,r3,r4 + /* Use the short loop if the src/dst addresses are not + * with the same offset of 8 bytes align boundary. + */ + xor r6,r3,r4 andi. r6,r6,7 - /* Use the short loop if length is less than 32B */ - cmpdi cr6,r5,31 + /* Fall back to short loop if compare at aligned addrs + * with less than 8 bytes. + */ + cmpdi cr6,r5,7 beq cr1,.Lzero - bne .Lshort - bgt cr6,.Llong + bgt cr6,.Lno_short .Lshort: mtctr r5 - 1: lbz rA,0(r3) lbz rB,0(r4) subf. rC,rB,rA @@ -78,11 +153,98 @@ _GLOBAL(memcmp) li r3,0 blr +.Lno_short: + dcbt 0,r3 + dcbt 0,r4 + bne .Ldiffoffset_8bytes_make_align_start + + +.Lsameoffset_8bytes_make_align_start: + /* attempt to compare bytes not aligned with 8 bytes so that + * rest comparison can run based on 8 bytes alignment. + */ + andi. r6,r3,7 + + /* Try to compare the first double word which is not 8 bytes aligned: + * load the first double word at (src & ~7UL) and shift left appropriate + * bits before comparision. + */ + rlwinm r6,r3,3,26,28 + beq .Lsameoffset_8bytes_aligned + clrrdi r3,r3,3 + clrrdi r4,r4,3 + LD rA,0,r3 + LD rB,0,r4 + sld rA,rA,r6 + sld rB,rB,r6 + cmpld cr0,rA,rB + srwi r6,r6,3 + bne cr0,.LcmpAB_lightweight + subfic r6,r6,8 + subf. r5,r6,r5 + addi r3,r3,8 + addi r4,r4,8 + beq .Lzero + +.Lsameoffset_8bytes_aligned: + /* now we are aligned with 8 bytes. + * Use .Llong loop if left cmp bytes are equal or greater than 32B. + */ + cmpdi cr6,r5,31 + bgt cr6,.Llong + +.Lcmp_lt32bytes: + /* compare 1 ~ 31 bytes, at least r3 addr is 8 bytes aligned now */ + cmpdi cr5,r5,7 + srdi r0,r5,3 + ble cr5,.Lcmp_rest_lt8bytes + + /* handle 8 ~ 31 bytes */ + clrldi r5,r5,61 + mtctr r0 +2: + LD rA,0,r3 + LD rB,0,r4 + cmpld cr0,rA,rB + addi r3,r3,8 + addi r4,r4,8 + bne cr0,.LcmpAB_lightweight + bdnz 2b + + cmpwi r5,0 + beq .Lzero + +.Lcmp_rest_lt8bytes: + /* Here we have only less than 8 bytes to compare with. at least s1 + * Address is aligned with 8 bytes. + * The next double words are load and shift right with appropriate + * bits. + */ + subfic r6,r5,8 + slwi r6,r6,3 + LD rA,0,r3 + LD rB,0,r4 + srd rA,rA,r6 + srd rB,rB,r6 + cmpld cr0,rA,rB + bne cr0,.LcmpAB_lightweight + b .Lzero + .Lnon_zero: mr r3,rC blr .Llong: +#ifdef CONFIG_ALTIVEC +BEGIN_FTR_SECTION + /* Try to use vmx loop if length is equal or greater than 4K */ + cmpldi cr6,r5,VMX_THRESH + bge cr6,.Lsameoffset_vmx_cmp +END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) + +.Llong_novmx_cmp: +#endif + /* At least s1 addr is aligned with 8 bytes */ li off8,8 li off16,16 li off24,24 @@ -232,4 +394,240 @@ _GLOBAL(memcmp) ld r28,-32(r1) ld r27,-40(r1) blr + +.LcmpAB_lightweight: /* skip NV GPRS restore */ + li r3,1 + bgtlr + li r3,-1 + blr + +#ifdef CONFIG_ALTIVEC +.Lsameoffset_vmx_cmp: + /* Enter with src/dst addrs has the same offset with 8 bytes + * align boundary. + * + * There is an optimization based on following fact: memcmp() + * prones to fail early at the first 32 bytes. + * Before applying VMX instructions which will lead to 32x128bits + * VMX regs load/restore penalty, we compare the first 32 bytes + * so that we can catch the ~80% fail cases. + */ + + li r0,4 + mtctr r0 +.Lsameoffset_prechk_32B_loop: + LD rA,0,r3 + LD rB,0,r4 + cmpld cr0,rA,rB + addi r3,r3,8 + addi r4,r4,8 + bne cr0,.LcmpAB_lightweight + addi r5,r5,-8 + bdnz .Lsameoffset_prechk_32B_loop + + ENTER_VMX_OPS + beq cr1,.Llong_novmx_cmp + +3: + /* need to check whether r4 has the same offset with r3 + * for 16 bytes boundary. + */ + xor r0,r3,r4 + andi. r0,r0,0xf + bne .Ldiffoffset_vmx_cmp_start + + /* len is no less than 4KB. Need to align with 16 bytes further. + */ + andi. rA,r3,8 + LD rA,0,r3 + beq 4f + LD rB,0,r4 + cmpld cr0,rA,rB + addi r3,r3,8 + addi r4,r4,8 + addi r5,r5,-8 + + beq cr0,4f + /* save and restore cr0 */ + mfocrf r5,128 + EXIT_VMX_OPS + mtocrf 128,r5 + b .LcmpAB_lightweight + +4: + /* compare 32 bytes for each loop */ + srdi r0,r5,5 + mtctr r0 + clrldi r5,r5,59 + li off16,16 + +.balign 16 +5: + lvx v0,0,r3 + lvx v1,0,r4 + VCMPEQUD_RC(v0,v0,v1) + bnl cr6,7f + lvx v0,off16,r3 + lvx v1,off16,r4 + VCMPEQUD_RC(v0,v0,v1) + bnl cr6,6f + addi r3,r3,32 + addi r4,r4,32 + bdnz 5b + + EXIT_VMX_OPS + cmpdi r5,0 + beq .Lzero + b .Lcmp_lt32bytes + +6: + addi r3,r3,16 + addi r4,r4,16 + +7: + /* diff the last 16 bytes */ + EXIT_VMX_OPS + LD rA,0,r3 + LD rB,0,r4 + cmpld cr0,rA,rB + li off8,8 + bne cr0,.LcmpAB_lightweight + + LD rA,off8,r3 + LD rB,off8,r4 + cmpld cr0,rA,rB + bne cr0,.LcmpAB_lightweight + b .Lzero +#endif + +.Ldiffoffset_8bytes_make_align_start: + /* now try to align s1 with 8 bytes */ + rlwinm r6,r3,3,26,28 + beq .Ldiffoffset_align_s1_8bytes + + clrrdi r3,r3,3 + LD rA,0,r3 + LD rB,0,r4 /* unaligned load */ + sld rA,rA,r6 + srd rA,rA,r6 + srd rB,rB,r6 + cmpld cr0,rA,rB + srwi r6,r6,3 + bne cr0,.LcmpAB_lightweight + + subfic r6,r6,8 + subf. r5,r6,r5 + addi r3,r3,8 + add r4,r4,r6 + + beq .Lzero + +.Ldiffoffset_align_s1_8bytes: + /* now s1 is aligned with 8 bytes. */ +#ifdef CONFIG_ALTIVEC +BEGIN_FTR_SECTION + /* only do vmx ops when the size equal or greater than 4K bytes */ + cmpdi cr5,r5,VMX_THRESH + bge cr5,.Ldiffoffset_vmx_cmp +END_FTR_SECTION_IFSET(CPU_FTR_ARCH_207S) + +.Ldiffoffset_novmx_cmp: +#endif + + + cmpdi cr5,r5,31 + ble cr5,.Lcmp_lt32bytes + +#ifdef CONFIG_ALTIVEC + b .Llong_novmx_cmp +#else + b .Llong +#endif + +#ifdef CONFIG_ALTIVEC +.Ldiffoffset_vmx_cmp: + /* perform a 32 bytes pre-checking before + * enable VMX operations. + */ + li r0,4 + mtctr r0 +.Ldiffoffset_prechk_32B_loop: + LD rA,0,r3 + LD rB,0,r4 + cmpld cr0,rA,rB + addi r3,r3,8 + addi r4,r4,8 + bne cr0,.LcmpAB_lightweight + addi r5,r5,-8 + bdnz .Ldiffoffset_prechk_32B_loop + + ENTER_VMX_OPS + beq cr1,.Ldiffoffset_novmx_cmp + +.Ldiffoffset_vmx_cmp_start: + /* Firstly try to align r3 with 16 bytes */ + andi. r6,r3,0xf + li off16,16 + beq .Ldiffoffset_vmx_s1_16bytes_align + + LVS v3,0,r3 + LVS v4,0,r4 + + lvx v5,0,r3 + lvx v6,0,r4 + LD_VSR_CROSS16B(r3,v3,v5,v7,v9) + LD_VSR_CROSS16B(r4,v4,v6,v8,v10) + + VCMPEQUB_RC(v7,v9,v10) + bnl cr6,.Ldiffoffset_vmx_diff_found + + subfic r6,r6,16 + subf r5,r6,r5 + add r3,r3,r6 + add r4,r4,r6 + +.Ldiffoffset_vmx_s1_16bytes_align: + /* now s1 is aligned with 16 bytes */ + lvx v6,0,r4 + LVS v4,0,r4 + srdi r6,r5,5 /* loop for 32 bytes each */ + clrldi r5,r5,59 + mtctr r6 + +.balign 16 +.Ldiffoffset_vmx_32bytesloop: + /* the first qw of r4 was saved in v6 */ + lvx v9,0,r3 + LD_VSR_CROSS16B(r4,v4,v6,v8,v10) + VCMPEQUB_RC(v7,v9,v10) + vor v6,v8,v8 + bnl cr6,.Ldiffoffset_vmx_diff_found + + addi r3,r3,16 + addi r4,r4,16 + + lvx v9,0,r3 + LD_VSR_CROSS16B(r4,v4,v6,v8,v10) + VCMPEQUB_RC(v7,v9,v10) + vor v6,v8,v8 + bnl cr6,.Ldiffoffset_vmx_diff_found + + addi r3,r3,16 + addi r4,r4,16 + + bdnz .Ldiffoffset_vmx_32bytesloop + + EXIT_VMX_OPS + + cmpdi r5,0 + beq .Lzero + b .Lcmp_lt32bytes + +.Ldiffoffset_vmx_diff_found: + EXIT_VMX_OPS + /* anyway, the diff will appear in next 16 bytes */ + li r5,16 + b .Lcmp_lt32bytes + +#endif EXPORT_SYMBOL(memcmp) diff --git a/arch/powerpc/lib/memcpy_64.S b/arch/powerpc/lib/memcpy_64.S index 8d8265be1a59..273ea67e60a1 100644 --- a/arch/powerpc/lib/memcpy_64.S +++ b/arch/powerpc/lib/memcpy_64.S @@ -9,6 +9,13 @@ #include <asm/processor.h> #include <asm/ppc_asm.h> #include <asm/export.h> +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> + +#ifndef SELFTEST_CASE +/* For big-endian, 0 == most CPUs, 1 == POWER6, 2 == Cell */ +#define SELFTEST_CASE 0 +#endif .align 7 _GLOBAL_TOC(memcpy) @@ -20,10 +27,8 @@ BEGIN_FTR_SECTION #endif FTR_SECTION_ELSE #ifdef CONFIG_PPC_BOOK3S_64 -#ifndef SELFTEST b memcpy_power7 #endif -#endif ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY) #ifdef __LITTLE_ENDIAN__ /* dumb little-endian memcpy that will get replaced at runtime */ @@ -47,6 +52,7 @@ ALT_FTR_SECTION_END_IFCLR(CPU_FTR_VMX_COPY) cleared. At the time of writing the only CPU that has this combination of bits set is Power6. */ +test_feature = (SELFTEST_CASE == 1) BEGIN_FTR_SECTION nop FTR_SECTION_ELSE @@ -55,6 +61,7 @@ ALT_FTR_SECTION_END(CPU_FTR_UNALIGNED_LD_STD | CPU_FTR_CP_USE_DCBTZ, \ CPU_FTR_UNALIGNED_LD_STD) .Ldst_aligned: addi r3,r3,-16 +test_feature = (SELFTEST_CASE == 0) BEGIN_FTR_SECTION andi. r0,r4,7 bne .Lsrc_unaligned diff --git a/arch/powerpc/lib/memcpy_power7.S b/arch/powerpc/lib/memcpy_power7.S index df7de9d3da08..89bfefcf7fcc 100644 --- a/arch/powerpc/lib/memcpy_power7.S +++ b/arch/powerpc/lib/memcpy_power7.S @@ -19,7 +19,10 @@ */ #include <asm/ppc_asm.h> -_GLOBAL(memcpy_power7) +#ifndef SELFTEST_CASE +/* 0 == don't use VMX, 1 == use VMX */ +#define SELFTEST_CASE 0 +#endif #ifdef __BIG_ENDIAN__ #define LVS(VRT,RA,RB) lvsl VRT,RA,RB @@ -29,20 +32,17 @@ _GLOBAL(memcpy_power7) #define VPERM(VRT,VRA,VRB,VRC) vperm VRT,VRB,VRA,VRC #endif -#ifdef CONFIG_ALTIVEC +_GLOBAL(memcpy_power7) cmpldi r5,16 cmpldi cr1,r5,4096 - std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) - blt .Lshort_copy - bgt cr1,.Lvmx_copy -#else - cmpldi r5,16 - - std r3,-STACKFRAMESIZE+STK_REG(R31)(r1) - blt .Lshort_copy +#ifdef CONFIG_ALTIVEC +test_feature = SELFTEST_CASE +BEGIN_FTR_SECTION + bgt cr1, .Lvmx_copy +END_FTR_SECTION_IFSET(CPU_FTR_ALTIVEC) #endif .Lnonvmx_copy: @@ -223,14 +223,14 @@ _GLOBAL(memcpy_power7) addi r1,r1,STACKFRAMESIZE b .Lnonvmx_copy -#ifdef CONFIG_ALTIVEC .Lvmx_copy: +#ifdef CONFIG_ALTIVEC mflr r0 std r4,-STACKFRAMESIZE+STK_REG(R30)(r1) std r5,-STACKFRAMESIZE+STK_REG(R29)(r1) std r0,16(r1) stdu r1,-STACKFRAMESIZE(r1) - bl enter_vmx_copy + bl enter_vmx_ops cmpwi cr1,r3,0 ld r0,STACKFRAMESIZE+16(r1) ld r3,STK_REG(R31)(r1) @@ -445,7 +445,7 @@ _GLOBAL(memcpy_power7) 15: addi r1,r1,STACKFRAMESIZE ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) - b exit_vmx_copy /* tail call optimise */ + b exit_vmx_ops /* tail call optimise */ .Lvmx_unaligned_copy: /* Get the destination 16B aligned */ @@ -649,5 +649,5 @@ _GLOBAL(memcpy_power7) 15: addi r1,r1,STACKFRAMESIZE ld r3,-STACKFRAMESIZE+STK_REG(R31)(r1) - b exit_vmx_copy /* tail call optimise */ + b exit_vmx_ops /* tail call optimise */ #endif /* CONFIG_ALTIVEC */ diff --git a/arch/powerpc/lib/strlen_32.S b/arch/powerpc/lib/strlen_32.S new file mode 100644 index 000000000000..0a8d3f64d493 --- /dev/null +++ b/arch/powerpc/lib/strlen_32.S @@ -0,0 +1,78 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* + * strlen() for PPC32 + * + * Copyright (C) 2018 Christophe Leroy CS Systemes d'Information. + * + * Inspired from glibc implementation + */ +#include <asm/ppc_asm.h> +#include <asm/export.h> +#include <asm/cache.h> + + .text + +/* + * Algorithm: + * + * 1) Given a word 'x', we can test to see if it contains any 0 bytes + * by subtracting 0x01010101, and seeing if any of the high bits of each + * byte changed from 0 to 1. This works because the least significant + * 0 byte must have had no incoming carry (otherwise it's not the least + * significant), so it is 0x00 - 0x01 == 0xff. For all other + * byte values, either they have the high bit set initially, or when + * 1 is subtracted you get a value in the range 0x00-0x7f, none of which + * have their high bit set. The expression here is + * (x - 0x01010101) & ~x & 0x80808080), which gives 0x00000000 when + * there were no 0x00 bytes in the word. You get 0x80 in bytes that + * match, but possibly false 0x80 matches in the next more significant + * byte to a true match due to carries. For little-endian this is + * of no consequence since the least significant match is the one + * we're interested in, but big-endian needs method 2 to find which + * byte matches. + * 2) Given a word 'x', we can test to see _which_ byte was zero by + * calculating ~(((x & ~0x80808080) - 0x80808080 - 1) | x | ~0x80808080). + * This produces 0x80 in each byte that was zero, and 0x00 in all + * the other bytes. The '| ~0x80808080' clears the low 7 bits in each + * byte, and the '| x' part ensures that bytes with the high bit set + * produce 0x00. The addition will carry into the high bit of each byte + * iff that byte had one of its low 7 bits set. We can then just see + * which was the most significant bit set and divide by 8 to find how + * many to add to the index. + * This is from the book 'The PowerPC Compiler Writer's Guide', + * by Steve Hoxey, Faraydon Karim, Bill Hay and Hank Warren. + */ + +_GLOBAL(strlen) + andi. r0, r3, 3 + lis r7, 0x0101 + addi r10, r3, -4 + addic r7, r7, 0x0101 /* r7 = 0x01010101 (lomagic) & clear XER[CA] */ + rotlwi r6, r7, 31 /* r6 = 0x80808080 (himagic) */ + bne- 3f + .balign IFETCH_ALIGN_BYTES +1: lwzu r9, 4(r10) +2: subf r8, r7, r9 + and. r8, r8, r6 + beq+ 1b + andc. r8, r8, r9 + beq+ 1b + andc r8, r9, r6 + orc r9, r9, r6 + subfe r8, r6, r8 + nor r8, r8, r9 + cntlzw r8, r8 + subf r3, r3, r10 + srwi r8, r8, 3 + add r3, r3, r8 + blr + + /* Missaligned string: make sure bytes before string are seen not 0 */ +3: xor r10, r10, r0 + orc r8, r8, r8 + lwzu r9, 4(r10) + slwi r0, r0, 3 + srw r8, r8, r0 + orc r9, r9, r8 + b 2b +EXPORT_SYMBOL(strlen) diff --git a/arch/powerpc/lib/vmx-helper.c b/arch/powerpc/lib/vmx-helper.c index bf925cdcaca9..9f340494a8ac 100644 --- a/arch/powerpc/lib/vmx-helper.c +++ b/arch/powerpc/lib/vmx-helper.c @@ -53,7 +53,7 @@ int exit_vmx_usercopy(void) return 0; } -int enter_vmx_copy(void) +int enter_vmx_ops(void) { if (in_interrupt()) return 0; @@ -70,7 +70,7 @@ int enter_vmx_copy(void) * passed a pointer to the destination which we return as required by a * memcpy implementation. */ -void *exit_vmx_copy(void *dest) +void *exit_vmx_ops(void *dest) { disable_kernel_altivec(); preempt_enable(); diff --git a/arch/powerpc/mm/44x_mmu.c b/arch/powerpc/mm/44x_mmu.c index 82b1ff759e26..12d92518e898 100644 --- a/arch/powerpc/mm/44x_mmu.c +++ b/arch/powerpc/mm/44x_mmu.c @@ -229,7 +229,7 @@ void setup_initial_memory_limit(phys_addr_t first_memblock_base, } #ifdef CONFIG_SMP -void mmu_init_secondary(int cpu) +void __init mmu_init_secondary(int cpu) { unsigned long addr; unsigned long memstart = memstart_addr & ~(PPC_PIN_SIZE - 1); diff --git a/arch/powerpc/mm/Makefile b/arch/powerpc/mm/Makefile index f06f3577d8d1..cdf6a9960046 100644 --- a/arch/powerpc/mm/Makefile +++ b/arch/powerpc/mm/Makefile @@ -19,7 +19,7 @@ obj-$(CONFIG_PPC_BOOK3S_64) += pgtable-hash64.o hash_utils_64.o slb_low.o slb.o obj-$(CONFIG_PPC_RADIX_MMU) += pgtable-radix.o tlb-radix.o obj-$(CONFIG_PPC_STD_MMU_32) += ppc_mmu_32.o hash_low_32.o mmu_context_hash32.o obj-$(CONFIG_PPC_STD_MMU) += tlb_hash$(BITS).o -ifeq ($(CONFIG_PPC_BOOK3S_64),y) +ifdef CONFIG_PPC_BOOK3S_64 obj-$(CONFIG_PPC_4K_PAGES) += hash64_4k.o obj-$(CONFIG_PPC_64K_PAGES) += hash64_64k.o endif @@ -31,7 +31,7 @@ obj-$(CONFIG_NEED_MULTIPLE_NODES) += numa.o obj-$(CONFIG_PPC_SPLPAR) += vphn.o obj-$(CONFIG_PPC_MM_SLICES) += slice.o obj-y += hugetlbpage.o -ifeq ($(CONFIG_HUGETLB_PAGE),y) +ifdef CONFIG_HUGETLB_PAGE obj-$(CONFIG_PPC_BOOK3S_64) += hugetlbpage-hash64.o obj-$(CONFIG_PPC_RADIX_MMU) += hugetlbpage-radix.o obj-$(CONFIG_PPC_BOOK3E_MMU) += hugetlbpage-book3e.o diff --git a/arch/powerpc/mm/dump_hashpagetable.c b/arch/powerpc/mm/dump_hashpagetable.c index 14cfb11b09d0..869294695048 100644 --- a/arch/powerpc/mm/dump_hashpagetable.c +++ b/arch/powerpc/mm/dump_hashpagetable.c @@ -19,7 +19,6 @@ #include <linux/mm.h> #include <linux/sched.h> #include <linux/seq_file.h> -#include <asm/fixmap.h> #include <asm/pgtable.h> #include <linux/const.h> #include <asm/page.h> @@ -260,7 +259,7 @@ static int pseries_find(unsigned long ea, int psize, bool primary, u64 *v, u64 * /* to check in the secondary hash table, we invert the hash */ if (!primary) hash = ~hash; - hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; /* see if we can find an entry in the hpte with this hash */ for (i = 0; i < HPTES_PER_GROUP; i += 4, hpte_group += 4) { lpar_rc = plpar_pte_read_4(0, hpte_group, (void *)ptes); diff --git a/arch/powerpc/mm/fault.c b/arch/powerpc/mm/fault.c index b1ca7a0974e3..7d262c6437c4 100644 --- a/arch/powerpc/mm/fault.c +++ b/arch/powerpc/mm/fault.c @@ -42,7 +42,6 @@ #include <asm/pgtable.h> #include <asm/mmu.h> #include <asm/mmu_context.h> -#include <asm/tlbflush.h> #include <asm/siginfo.h> #include <asm/debug.h> diff --git a/arch/powerpc/mm/hash64_4k.c b/arch/powerpc/mm/hash64_4k.c index d573d7d07f25..6fa6765a10eb 100644 --- a/arch/powerpc/mm/hash64_4k.c +++ b/arch/powerpc/mm/hash64_4k.c @@ -80,7 +80,7 @@ int __hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, hash = hpt_hash(vpn, shift, ssize); repeat: - hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; /* Insert into the hash table, primary slot */ slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0, @@ -89,7 +89,7 @@ repeat: * Primary is full, try the secondary */ if (unlikely(slot == -1)) { - hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP; slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, HPTE_V_SECONDARY, @@ -97,8 +97,8 @@ repeat: MMU_PAGE_4K, ssize); if (slot == -1) { if (mftb() & 0x1) - hpte_group = ((hash & htab_hash_mask) * - HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * + HPTES_PER_GROUP; mmu_hash_ops.hpte_remove(hpte_group); /* * FIXME!! Should be try the group from which we removed ? diff --git a/arch/powerpc/mm/hash64_64k.c b/arch/powerpc/mm/hash64_64k.c index e601d95c3b20..3afa253d7f52 100644 --- a/arch/powerpc/mm/hash64_64k.c +++ b/arch/powerpc/mm/hash64_64k.c @@ -154,7 +154,7 @@ htab_insert_hpte: } hash = hpt_hash(vpn, shift, ssize); repeat: - hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; /* Insert into the hash table, primary slot */ slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0, @@ -165,7 +165,7 @@ repeat: if (unlikely(slot == -1)) { bool soft_invalid; - hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP; slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, HPTE_V_SECONDARY, MMU_PAGE_4K, MMU_PAGE_4K, @@ -193,8 +193,7 @@ repeat: * that we do not get the same soft-invalid slot. */ if (soft_invalid || (mftb() & 0x1)) - hpte_group = ((hash & htab_hash_mask) * - HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; mmu_hash_ops.hpte_remove(hpte_group); /* @@ -288,7 +287,7 @@ int __hash_page_64K(unsigned long ea, unsigned long access, hash = hpt_hash(vpn, shift, ssize); repeat: - hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; /* Insert into the hash table, primary slot */ slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0, @@ -298,7 +297,7 @@ repeat: * Primary is full, try the secondary */ if (unlikely(slot == -1)) { - hpte_group = ((~hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP; slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, HPTE_V_SECONDARY, @@ -306,8 +305,8 @@ repeat: MMU_PAGE_64K, ssize); if (slot == -1) { if (mftb() & 0x1) - hpte_group = ((hash & htab_hash_mask) * - HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * + HPTES_PER_GROUP; mmu_hash_ops.hpte_remove(hpte_group); /* * FIXME!! Should be try the group from which we removed ? diff --git a/arch/powerpc/mm/hash_low_32.S b/arch/powerpc/mm/hash_low_32.S index ffbd7c0bda96..26acf6c8c20c 100644 --- a/arch/powerpc/mm/hash_low_32.S +++ b/arch/powerpc/mm/hash_low_32.S @@ -27,6 +27,7 @@ #include <asm/thread_info.h> #include <asm/asm-offsets.h> #include <asm/export.h> +#include <asm/feature-fixups.h> #ifdef CONFIG_SMP .section .bss diff --git a/arch/powerpc/mm/hash_native_64.c b/arch/powerpc/mm/hash_native_64.c index 1d049c78c82a..729f02df8290 100644 --- a/arch/powerpc/mm/hash_native_64.c +++ b/arch/powerpc/mm/hash_native_64.c @@ -23,13 +23,13 @@ #include <asm/mmu.h> #include <asm/mmu_context.h> #include <asm/pgtable.h> -#include <asm/tlbflush.h> #include <asm/trace.h> #include <asm/tlb.h> #include <asm/cputable.h> #include <asm/udbg.h> #include <asm/kexec.h> #include <asm/ppc-opcode.h> +#include <asm/feature-fixups.h> #include <misc/cxl-base.h> @@ -423,9 +423,7 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)", vpn, want_v & HPTE_V_AVPN, slot, newpp); - hpte_v = be64_to_cpu(hptep->v); - if (cpu_has_feature(CPU_FTR_ARCH_300)) - hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); + hpte_v = hpte_get_old_v(hptep); /* * We need to invalidate the TLB always because hpte_remove doesn't do * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less @@ -439,9 +437,7 @@ static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, } else { native_lock_hpte(hptep); /* recheck with locks held */ - hpte_v = be64_to_cpu(hptep->v); - if (cpu_has_feature(CPU_FTR_ARCH_300)) - hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); + hpte_v = hpte_get_old_v(hptep); if (unlikely(!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID))) { ret = -1; @@ -481,11 +477,9 @@ static long native_hpte_find(unsigned long vpn, int psize, int ssize) /* Bolted mappings are only ever in the primary group */ slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; for (i = 0; i < HPTES_PER_GROUP; i++) { - hptep = htab_address + slot; - hpte_v = be64_to_cpu(hptep->v); - if (cpu_has_feature(CPU_FTR_ARCH_300)) - hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); + hptep = htab_address + slot; + hpte_v = hpte_get_old_v(hptep); if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) /* HPTE matches */ return slot; @@ -574,11 +568,19 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot); want_v = hpte_encode_avpn(vpn, bpsize, ssize); - native_lock_hpte(hptep); - hpte_v = be64_to_cpu(hptep->v); - if (cpu_has_feature(CPU_FTR_ARCH_300)) - hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); + hpte_v = hpte_get_old_v(hptep); + if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) { + native_lock_hpte(hptep); + /* recheck with locks held */ + hpte_v = hpte_get_old_v(hptep); + + if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) + /* Invalidate the hpte. NOTE: this also unlocks it */ + hptep->v = 0; + else + native_unlock_hpte(hptep); + } /* * We need to invalidate the TLB always because hpte_remove doesn't do * a tlb invalidate. If a hash bucket gets full, we "evict" a more/less @@ -586,13 +588,6 @@ static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, * (hpte_remove) because we assume the old translation is still * technically "valid". */ - if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) - native_unlock_hpte(hptep); - else - /* Invalidate the hpte. NOTE: this also unlocks it */ - hptep->v = 0; - - /* Invalidate the TLB */ tlbie(vpn, bpsize, apsize, ssize, local); local_irq_restore(flags); @@ -634,17 +629,23 @@ static void native_hugepage_invalidate(unsigned long vsid, hptep = htab_address + slot; want_v = hpte_encode_avpn(vpn, psize, ssize); - native_lock_hpte(hptep); - hpte_v = be64_to_cpu(hptep->v); - if (cpu_has_feature(CPU_FTR_ARCH_300)) - hpte_v = hpte_new_to_old_v(hpte_v, be64_to_cpu(hptep->r)); + hpte_v = hpte_get_old_v(hptep); /* Even if we miss, we need to invalidate the TLB */ - if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) - native_unlock_hpte(hptep); - else - /* Invalidate the hpte. NOTE: this also unlocks it */ - hptep->v = 0; + if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) { + /* recheck with locks held */ + native_lock_hpte(hptep); + hpte_v = hpte_get_old_v(hptep); + + if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) { + /* + * Invalidate the hpte. NOTE: this also unlocks it + */ + + hptep->v = 0; + } else + native_unlock_hpte(hptep); + } /* * We need to do tlb invalidate for all the address, tlbie * instruction compares entry_VA in tlb with the VA specified @@ -812,16 +813,19 @@ static void native_flush_hash_range(unsigned long number, int local) slot += hidx & _PTEIDX_GROUP_IX; hptep = htab_address + slot; want_v = hpte_encode_avpn(vpn, psize, ssize); + hpte_v = hpte_get_old_v(hptep); + + if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) + continue; + /* lock and try again */ native_lock_hpte(hptep); - hpte_v = be64_to_cpu(hptep->v); - if (cpu_has_feature(CPU_FTR_ARCH_300)) - hpte_v = hpte_new_to_old_v(hpte_v, - be64_to_cpu(hptep->r)); - if (!HPTE_V_COMPARE(hpte_v, want_v) || - !(hpte_v & HPTE_V_VALID)) + hpte_v = hpte_get_old_v(hptep); + + if (!HPTE_V_COMPARE(hpte_v, want_v) || !(hpte_v & HPTE_V_VALID)) native_unlock_hpte(hptep); else hptep->v = 0; + } pte_iterate_hashed_end(); } diff --git a/arch/powerpc/mm/hash_utils_64.c b/arch/powerpc/mm/hash_utils_64.c index 8318716e5075..f23a89d8e4ce 100644 --- a/arch/powerpc/mm/hash_utils_64.c +++ b/arch/powerpc/mm/hash_utils_64.c @@ -48,7 +48,6 @@ #include <linux/uaccess.h> #include <asm/machdep.h> #include <asm/prom.h> -#include <asm/tlbflush.h> #include <asm/io.h> #include <asm/eeh.h> #include <asm/tlb.h> @@ -808,31 +807,6 @@ int hash__remove_section_mapping(unsigned long start, unsigned long end) } #endif /* CONFIG_MEMORY_HOTPLUG */ -static void update_hid_for_hash(void) -{ - unsigned long hid0; - unsigned long rb = 3UL << PPC_BITLSHIFT(53); /* IS = 3 */ - - asm volatile("ptesync": : :"memory"); - /* prs = 0, ric = 2, rs = 0, r = 1 is = 3 */ - asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) - : : "r"(rb), "i"(0), "i"(0), "i"(2), "r"(0) : "memory"); - asm volatile("eieio; tlbsync; ptesync; isync; slbia": : :"memory"); - trace_tlbie(0, 0, rb, 0, 2, 0, 0); - - /* - * now switch the HID - */ - hid0 = mfspr(SPRN_HID0); - hid0 &= ~HID0_POWER9_RADIX; - mtspr(SPRN_HID0, hid0); - asm volatile("isync": : :"memory"); - - /* Wait for it to happen */ - while ((mfspr(SPRN_HID0) & HID0_POWER9_RADIX)) - cpu_relax(); -} - static void __init hash_init_partition_table(phys_addr_t hash_table, unsigned long htab_size) { @@ -845,8 +819,6 @@ static void __init hash_init_partition_table(phys_addr_t hash_table, htab_size = __ilog2(htab_size) - 18; mmu_partition_table_set_entry(0, hash_table | htab_size, 0); pr_info("Partition table %p\n", partition_tb); - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - update_hid_for_hash(); } static void __init htab_initialize(void) @@ -1077,9 +1049,6 @@ void hash__early_init_mmu_secondary(void) /* Initialize hash table for that CPU */ if (!firmware_has_feature(FW_FEATURE_LPAR)) { - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - update_hid_for_hash(); - if (!cpu_has_feature(CPU_FTR_ARCH_300)) mtspr(SPRN_SDR1, _SDR1); else @@ -1783,8 +1752,7 @@ long hpte_insert_repeating(unsigned long hash, unsigned long vpn, long slot; repeat: - hpte_group = ((hash & htab_hash_mask) * - HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; /* Insert into the hash table, primary slot */ slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, vflags, @@ -1792,15 +1760,14 @@ repeat: /* Primary is full, try the secondary */ if (unlikely(slot == -1)) { - hpte_group = ((~hash & htab_hash_mask) * - HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP; slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, vflags | HPTE_V_SECONDARY, psize, psize, ssize); if (slot == -1) { if (mftb() & 0x1) - hpte_group = ((hash & htab_hash_mask) * - HPTES_PER_GROUP)&~0x7UL; + hpte_group = (hash & htab_hash_mask) * + HPTES_PER_GROUP; mmu_hash_ops.hpte_remove(hpte_group); goto repeat; diff --git a/arch/powerpc/mm/highmem.c b/arch/powerpc/mm/highmem.c index 668e87d03f9e..82a0e37557a5 100644 --- a/arch/powerpc/mm/highmem.c +++ b/arch/powerpc/mm/highmem.c @@ -56,7 +56,7 @@ EXPORT_SYMBOL(kmap_atomic_prot); void __kunmap_atomic(void *kvaddr) { unsigned long vaddr = (unsigned long) kvaddr & PAGE_MASK; - int type; + int type __maybe_unused; if (vaddr < __fix_to_virt(FIX_KMAP_END)) { pagefault_enable(); diff --git a/arch/powerpc/mm/hugepage-hash64.c b/arch/powerpc/mm/hugepage-hash64.c index f20d16f849c5..01f213d2bcb9 100644 --- a/arch/powerpc/mm/hugepage-hash64.c +++ b/arch/powerpc/mm/hugepage-hash64.c @@ -128,7 +128,7 @@ int __hash_page_thp(unsigned long ea, unsigned long access, unsigned long vsid, new_pmd |= H_PAGE_HASHPTE; repeat: - hpte_group = ((hash & htab_hash_mask) * HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * HPTES_PER_GROUP; /* Insert into the hash table, primary slot */ slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, 0, @@ -137,16 +137,15 @@ repeat: * Primary is full, try the secondary */ if (unlikely(slot == -1)) { - hpte_group = ((~hash & htab_hash_mask) * - HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (~hash & htab_hash_mask) * HPTES_PER_GROUP; slot = mmu_hash_ops.hpte_insert(hpte_group, vpn, pa, rflags, HPTE_V_SECONDARY, psize, lpsize, ssize); if (slot == -1) { if (mftb() & 0x1) - hpte_group = ((hash & htab_hash_mask) * - HPTES_PER_GROUP) & ~0x7UL; + hpte_group = (hash & htab_hash_mask) * + HPTES_PER_GROUP; mmu_hash_ops.hpte_remove(hpte_group); goto repeat; diff --git a/arch/powerpc/mm/hugetlbpage.c b/arch/powerpc/mm/hugetlbpage.c index 8a9a49c13865..e87f9ef9115b 100644 --- a/arch/powerpc/mm/hugetlbpage.c +++ b/arch/powerpc/mm/hugetlbpage.c @@ -118,15 +118,6 @@ static int __hugepte_alloc(struct mm_struct *mm, hugepd_t *hpdp, } /* - * These macros define how to determine which level of the page table holds - * the hpdp. - */ -#if defined(CONFIG_PPC_FSL_BOOK3E) || defined(CONFIG_PPC_8xx) -#define HUGEPD_PGD_SHIFT PGDIR_SHIFT -#define HUGEPD_PUD_SHIFT PUD_SHIFT -#endif - -/* * At this point we do the placement change only for BOOK3S 64. This would * possibly work on other subarchs. */ @@ -174,13 +165,13 @@ pte_t *huge_pte_alloc(struct mm_struct *mm, unsigned long addr, unsigned long sz } } #else - if (pshift >= HUGEPD_PGD_SHIFT) { + if (pshift >= PGDIR_SHIFT) { ptl = &mm->page_table_lock; hpdp = (hugepd_t *)pg; } else { pdshift = PUD_SHIFT; pu = pud_alloc(mm, pg, addr); - if (pshift >= HUGEPD_PUD_SHIFT) { + if (pshift >= PUD_SHIFT) { ptl = pud_lockptr(mm, pu); hpdp = (hugepd_t *)pu; } else { @@ -621,15 +612,12 @@ static int __init add_huge_page_size(unsigned long long size) * firmware we only add hugetlb support for page sizes that can be * supported by linux page table layout. * For now we have - * Radix: 2M + * Radix: 2M and 1G * Hash: 16M and 16G */ if (radix_enabled()) { - if (mmu_psize != MMU_PAGE_2M) { - if (cpu_has_feature(CPU_FTR_POWER9_DD1) || - (mmu_psize != MMU_PAGE_1G)) - return -EINVAL; - } + if (mmu_psize != MMU_PAGE_2M && mmu_psize != MMU_PAGE_1G) + return -EINVAL; } else { if (mmu_psize != MMU_PAGE_16M && mmu_psize != MMU_PAGE_16G) return -EINVAL; @@ -695,9 +683,9 @@ static int __init hugetlbpage_init(void) else pdshift = PMD_SHIFT; #else - if (shift < HUGEPD_PUD_SHIFT) + if (shift < PUD_SHIFT) pdshift = PMD_SHIFT; - else if (shift < HUGEPD_PGD_SHIFT) + else if (shift < PGDIR_SHIFT) pdshift = PUD_SHIFT; else pdshift = PGDIR_SHIFT; diff --git a/arch/powerpc/mm/mmu_context_book3s64.c b/arch/powerpc/mm/mmu_context_book3s64.c index f3d4b4a0e561..4a892d894a0f 100644 --- a/arch/powerpc/mm/mmu_context_book3s64.c +++ b/arch/powerpc/mm/mmu_context_book3s64.c @@ -200,9 +200,9 @@ static void pte_frag_destroy(void *pte_frag) /* drop all the pending references */ count = ((unsigned long)pte_frag & ~PAGE_MASK) >> PTE_FRAG_SIZE_SHIFT; /* We allow PTE_FRAG_NR fragments from a PTE page */ - if (page_ref_sub_and_test(page, PTE_FRAG_NR - count)) { + if (atomic_sub_and_test(PTE_FRAG_NR - count, &page->pt_frag_refcount)) { pgtable_page_dtor(page); - free_unref_page(page); + __free_page(page); } } @@ -215,13 +215,13 @@ static void pmd_frag_destroy(void *pmd_frag) /* drop all the pending references */ count = ((unsigned long)pmd_frag & ~PAGE_MASK) >> PMD_FRAG_SIZE_SHIFT; /* We allow PTE_FRAG_NR fragments from a PTE page */ - if (page_ref_sub_and_test(page, PMD_FRAG_NR - count)) { + if (atomic_sub_and_test(PMD_FRAG_NR - count, &page->pt_frag_refcount)) { pgtable_pmd_page_dtor(page); - free_unref_page(page); + __free_page(page); } } -static void destroy_pagetable_page(struct mm_struct *mm) +static void destroy_pagetable_cache(struct mm_struct *mm) { void *frag; @@ -244,13 +244,14 @@ void destroy_context(struct mm_struct *mm) WARN_ON(process_tb[mm->context.id].prtb0 != 0); else subpage_prot_free(mm); - destroy_pagetable_page(mm); destroy_contexts(&mm->context); mm->context.id = MMU_NO_CONTEXT; } void arch_exit_mmap(struct mm_struct *mm) { + destroy_pagetable_cache(mm); + if (radix_enabled()) { /* * Radix doesn't have a valid bit in the process table @@ -273,15 +274,7 @@ void arch_exit_mmap(struct mm_struct *mm) #ifdef CONFIG_PPC_RADIX_MMU void radix__switch_mmu_context(struct mm_struct *prev, struct mm_struct *next) { - - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { - isync(); - mtspr(SPRN_PID, next->context.id); - isync(); - asm volatile(PPC_INVALIDATE_ERAT : : :"memory"); - } else { - mtspr(SPRN_PID, next->context.id); - isync(); - } + mtspr(SPRN_PID, next->context.id); + isync(); } #endif diff --git a/arch/powerpc/mm/mmu_context_hash32.c b/arch/powerpc/mm/mmu_context_hash32.c index aa5a7fd89461..921c1e33e941 100644 --- a/arch/powerpc/mm/mmu_context_hash32.c +++ b/arch/powerpc/mm/mmu_context_hash32.c @@ -27,7 +27,6 @@ #include <linux/export.h> #include <asm/mmu_context.h> -#include <asm/tlbflush.h> /* * On 32-bit PowerPC 6xx/7xx/7xxx CPUs, we use a set of 16 VSIDs diff --git a/arch/powerpc/mm/mmu_decl.h b/arch/powerpc/mm/mmu_decl.h index c4c0a09a7775..e5d779eed181 100644 --- a/arch/powerpc/mm/mmu_decl.h +++ b/arch/powerpc/mm/mmu_decl.h @@ -19,7 +19,6 @@ * */ #include <linux/mm.h> -#include <asm/tlbflush.h> #include <asm/mmu.h> #ifdef CONFIG_PPC_MMU_NOHASH diff --git a/arch/powerpc/mm/pgtable-book3s64.c b/arch/powerpc/mm/pgtable-book3s64.c index 4afbfbb64bfd..01d7c0f7c4f0 100644 --- a/arch/powerpc/mm/pgtable-book3s64.c +++ b/arch/powerpc/mm/pgtable-book3s64.c @@ -270,6 +270,8 @@ static pmd_t *__alloc_for_pmdcache(struct mm_struct *mm) return NULL; } + atomic_set(&page->pt_frag_refcount, 1); + ret = page_address(page); /* * if we support only one fragment just return the @@ -285,7 +287,7 @@ static pmd_t *__alloc_for_pmdcache(struct mm_struct *mm) * count. */ if (likely(!mm->context.pmd_frag)) { - set_page_count(page, PMD_FRAG_NR); + atomic_set(&page->pt_frag_refcount, PMD_FRAG_NR); mm->context.pmd_frag = ret + PMD_FRAG_SIZE; } spin_unlock(&mm->page_table_lock); @@ -308,9 +310,10 @@ void pmd_fragment_free(unsigned long *pmd) { struct page *page = virt_to_page(pmd); - if (put_page_testzero(page)) { + BUG_ON(atomic_read(&page->pt_frag_refcount) <= 0); + if (atomic_dec_and_test(&page->pt_frag_refcount)) { pgtable_pmd_page_dtor(page); - free_unref_page(page); + __free_page(page); } } @@ -352,6 +355,7 @@ static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel) return NULL; } + atomic_set(&page->pt_frag_refcount, 1); ret = page_address(page); /* @@ -367,7 +371,7 @@ static pte_t *__alloc_for_ptecache(struct mm_struct *mm, int kernel) * count. */ if (likely(!mm->context.pte_frag)) { - set_page_count(page, PTE_FRAG_NR); + atomic_set(&page->pt_frag_refcount, PTE_FRAG_NR); mm->context.pte_frag = ret + PTE_FRAG_SIZE; } spin_unlock(&mm->page_table_lock); @@ -390,10 +394,11 @@ void pte_fragment_free(unsigned long *table, int kernel) { struct page *page = virt_to_page(table); - if (put_page_testzero(page)) { + BUG_ON(atomic_read(&page->pt_frag_refcount) <= 0); + if (atomic_dec_and_test(&page->pt_frag_refcount)) { if (!kernel) pgtable_page_dtor(page); - free_unref_page(page); + __free_page(page); } } @@ -450,3 +455,25 @@ void pgtable_free_tlb(struct mmu_gather *tlb, void *table, int index) return pgtable_free(table, index); } #endif + +#ifdef CONFIG_PROC_FS +atomic_long_t direct_pages_count[MMU_PAGE_COUNT]; + +void arch_report_meminfo(struct seq_file *m) +{ + /* + * Hash maps the memory with one size mmu_linear_psize. + * So don't bother to print these on hash + */ + if (!radix_enabled()) + return; + seq_printf(m, "DirectMap4k: %8lu kB\n", + atomic_long_read(&direct_pages_count[MMU_PAGE_4K]) << 2); + seq_printf(m, "DirectMap64k: %8lu kB\n", + atomic_long_read(&direct_pages_count[MMU_PAGE_64K]) << 6); + seq_printf(m, "DirectMap2M: %8lu kB\n", + atomic_long_read(&direct_pages_count[MMU_PAGE_2M]) << 11); + seq_printf(m, "DirectMap1G: %8lu kB\n", + atomic_long_read(&direct_pages_count[MMU_PAGE_1G]) << 20); +} +#endif /* CONFIG_PROC_FS */ diff --git a/arch/powerpc/mm/pgtable-radix.c b/arch/powerpc/mm/pgtable-radix.c index 96f68c5aa1f5..7be99fd9af15 100644 --- a/arch/powerpc/mm/pgtable-radix.c +++ b/arch/powerpc/mm/pgtable-radix.c @@ -226,16 +226,6 @@ void radix__mark_rodata_ro(void) { unsigned long start, end; - /* - * mark_rodata_ro() will mark itself as !writable at some point. - * Due to DD1 workaround in radix__pte_update(), we'll end up with - * an invalid pte and the system will crash quite severly. - */ - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { - pr_warn("Warning: Unable to mark rodata read only on P9 DD1\n"); - return; - } - start = (unsigned long)_stext; end = (unsigned long)__init_begin; @@ -277,6 +267,7 @@ static int __meminit create_physical_mapping(unsigned long start, #else int split_text_mapping = 0; #endif + int psize; start = _ALIGN_UP(start, PAGE_SIZE); for (addr = start; addr < end; addr += mapping_size) { @@ -290,13 +281,17 @@ static int __meminit create_physical_mapping(unsigned long start, retry: if (IS_ALIGNED(addr, PUD_SIZE) && gap >= PUD_SIZE && mmu_psize_defs[MMU_PAGE_1G].shift && - PUD_SIZE <= max_mapping_size) + PUD_SIZE <= max_mapping_size) { mapping_size = PUD_SIZE; - else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE && - mmu_psize_defs[MMU_PAGE_2M].shift) + psize = MMU_PAGE_1G; + } else if (IS_ALIGNED(addr, PMD_SIZE) && gap >= PMD_SIZE && + mmu_psize_defs[MMU_PAGE_2M].shift) { mapping_size = PMD_SIZE; - else + psize = MMU_PAGE_2M; + } else { mapping_size = PAGE_SIZE; + psize = mmu_virtual_psize; + } if (split_text_mapping && (mapping_size == PUD_SIZE) && (addr <= __pa_symbol(__init_begin)) && @@ -307,8 +302,10 @@ retry: if (split_text_mapping && (mapping_size == PMD_SIZE) && (addr <= __pa_symbol(__init_begin)) && - (addr + mapping_size) >= __pa_symbol(_stext)) + (addr + mapping_size) >= __pa_symbol(_stext)) { mapping_size = PAGE_SIZE; + psize = mmu_virtual_psize; + } if (mapping_size != previous_size) { print_mapping(start, addr, previous_size); @@ -326,6 +323,8 @@ retry: rc = __map_kernel_page(vaddr, addr, prot, mapping_size, nid, start, end); if (rc) return rc; + + update_page_count(psize, 1); } print_mapping(start, addr, mapping_size); @@ -533,35 +532,6 @@ found: return; } -static void update_hid_for_radix(void) -{ - unsigned long hid0; - unsigned long rb = 3UL << PPC_BITLSHIFT(53); /* IS = 3 */ - - asm volatile("ptesync": : :"memory"); - /* prs = 0, ric = 2, rs = 0, r = 1 is = 3 */ - asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) - : : "r"(rb), "i"(1), "i"(0), "i"(2), "r"(0) : "memory"); - /* prs = 1, ric = 2, rs = 0, r = 1 is = 3 */ - asm volatile(PPC_TLBIE_5(%0, %4, %3, %2, %1) - : : "r"(rb), "i"(1), "i"(1), "i"(2), "r"(0) : "memory"); - asm volatile("eieio; tlbsync; ptesync; isync; slbia": : :"memory"); - trace_tlbie(0, 0, rb, 0, 2, 0, 1); - trace_tlbie(0, 0, rb, 0, 2, 1, 1); - - /* - * now switch the HID - */ - hid0 = mfspr(SPRN_HID0); - hid0 |= HID0_POWER9_RADIX; - mtspr(SPRN_HID0, hid0); - asm volatile("isync": : :"memory"); - - /* Wait for it to happen */ - while (!(mfspr(SPRN_HID0) & HID0_POWER9_RADIX)) - cpu_relax(); -} - static void radix_init_amor(void) { /* @@ -576,22 +546,12 @@ static void radix_init_amor(void) static void radix_init_iamr(void) { - unsigned long iamr; - - /* - * The IAMR should set to 0 on DD1. - */ - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - iamr = 0; - else - iamr = (1ul << 62); - /* * Radix always uses key0 of the IAMR to determine if an access is * allowed. We set bit 0 (IBM bit 1) of key0, to prevent instruction * fetch. */ - mtspr(SPRN_IAMR, iamr); + mtspr(SPRN_IAMR, (1ul << 62)); } void __init radix__early_init_mmu(void) @@ -644,8 +604,6 @@ void __init radix__early_init_mmu(void) if (!firmware_has_feature(FW_FEATURE_LPAR)) { radix_init_native(); - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - update_hid_for_radix(); lpcr = mfspr(SPRN_LPCR); mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR); radix_init_partition_table(); @@ -671,10 +629,6 @@ void radix__early_init_mmu_secondary(void) * update partition table control register and UPRT */ if (!firmware_has_feature(FW_FEATURE_LPAR)) { - - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - update_hid_for_radix(); - lpcr = mfspr(SPRN_LPCR); mtspr(SPRN_LPCR, lpcr | LPCR_UPRT | LPCR_HR); @@ -1095,8 +1049,7 @@ void radix__ptep_set_access_flags(struct vm_area_struct *vma, pte_t *ptep, * To avoid NMMU hang while relaxing access, we need mark * the pte invalid in between. */ - if (cpu_has_feature(CPU_FTR_POWER9_DD1) || - atomic_read(&mm->context.copros) > 0) { + if (atomic_read(&mm->context.copros) > 0) { unsigned long old_pte, new_pte; old_pte = __radix_pte_update(ptep, ~0, 0); diff --git a/arch/powerpc/mm/pkeys.c b/arch/powerpc/mm/pkeys.c index e6f500fabf5e..333b1f80c435 100644 --- a/arch/powerpc/mm/pkeys.c +++ b/arch/powerpc/mm/pkeys.c @@ -14,9 +14,12 @@ DEFINE_STATIC_KEY_TRUE(pkey_disabled); bool pkey_execute_disable_supported; int pkeys_total; /* Total pkeys as per device tree */ bool pkeys_devtree_defined; /* pkey property exported by device tree */ -u32 initial_allocation_mask; /* Bits set for reserved keys */ -u64 pkey_amr_uamor_mask; /* Bits in AMR/UMOR not to be touched */ +u32 initial_allocation_mask; /* Bits set for the initially allocated keys */ +u32 reserved_allocation_mask; /* Bits set for reserved keys */ +u64 pkey_amr_mask; /* Bits in AMR not to be touched */ u64 pkey_iamr_mask; /* Bits in AMR not to be touched */ +u64 pkey_uamor_mask; /* Bits in UMOR not to be touched */ +int execute_only_key = 2; #define AMR_BITS_PER_PKEY 2 #define AMR_RD_BIT 0x1UL @@ -91,7 +94,7 @@ int pkey_initialize(void) * arch-neutral code. */ pkeys_total = min_t(int, pkeys_total, - (ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)); + ((ARCH_VM_PKEY_FLAGS >> VM_PKEY_SHIFT)+1)); if (!pkey_mmu_enabled() || radix_enabled() || !pkeys_total) static_branch_enable(&pkey_disabled); @@ -119,20 +122,39 @@ int pkey_initialize(void) #else os_reserved = 0; #endif - initial_allocation_mask = ~0x0; - pkey_amr_uamor_mask = ~0x0ul; + /* Bits are in LE format. */ + reserved_allocation_mask = (0x1 << 1) | (0x1 << execute_only_key); + + /* register mask is in BE format */ + pkey_amr_mask = ~0x0ul; + pkey_amr_mask &= ~(0x3ul << pkeyshift(0)); + pkey_iamr_mask = ~0x0ul; - /* - * key 0, 1 are reserved. - * key 0 is the default key, which allows read/write/execute. - * key 1 is recommended not to be used. PowerISA(3.0) page 1015, - * programming note. - */ - for (i = 2; i < (pkeys_total - os_reserved); i++) { - initial_allocation_mask &= ~(0x1 << i); - pkey_amr_uamor_mask &= ~(0x3ul << pkeyshift(i)); - pkey_iamr_mask &= ~(0x1ul << pkeyshift(i)); + pkey_iamr_mask &= ~(0x3ul << pkeyshift(0)); + pkey_iamr_mask &= ~(0x3ul << pkeyshift(execute_only_key)); + + pkey_uamor_mask = ~0x0ul; + pkey_uamor_mask &= ~(0x3ul << pkeyshift(0)); + pkey_uamor_mask &= ~(0x3ul << pkeyshift(execute_only_key)); + + /* mark the rest of the keys as reserved and hence unavailable */ + for (i = (pkeys_total - os_reserved); i < pkeys_total; i++) { + reserved_allocation_mask |= (0x1 << i); + pkey_uamor_mask &= ~(0x3ul << pkeyshift(i)); + } + initial_allocation_mask = reserved_allocation_mask | (0x1 << 0); + + if (unlikely((pkeys_total - os_reserved) <= execute_only_key)) { + /* + * Insufficient number of keys to support + * execute only key. Mark it unavailable. + * Any AMR, UAMOR, IAMR bit set for + * this key is irrelevant since this key + * can never be allocated. + */ + execute_only_key = -1; } + return 0; } @@ -143,8 +165,7 @@ void pkey_mm_init(struct mm_struct *mm) if (static_branch_likely(&pkey_disabled)) return; mm_pkey_allocation_map(mm) = initial_allocation_mask; - /* -1 means unallocated or invalid */ - mm->context.execute_only_pkey = -1; + mm->context.execute_only_pkey = execute_only_key; } static inline u64 read_amr(void) @@ -213,33 +234,6 @@ static inline void init_iamr(int pkey, u8 init_bits) write_iamr(old_iamr | new_iamr_bits); } -static void pkey_status_change(int pkey, bool enable) -{ - u64 old_uamor; - - /* Reset the AMR and IAMR bits for this key */ - init_amr(pkey, 0x0); - init_iamr(pkey, 0x0); - - /* Enable/disable key */ - old_uamor = read_uamor(); - if (enable) - old_uamor |= (0x3ul << pkeyshift(pkey)); - else - old_uamor &= ~(0x3ul << pkeyshift(pkey)); - write_uamor(old_uamor); -} - -void __arch_activate_pkey(int pkey) -{ - pkey_status_change(pkey, true); -} - -void __arch_deactivate_pkey(int pkey) -{ - pkey_status_change(pkey, false); -} - /* * Set the access rights in AMR IAMR and UAMOR registers for @pkey to that * specified in @init_val. @@ -289,9 +283,6 @@ void thread_pkey_regs_restore(struct thread_struct *new_thread, if (static_branch_likely(&pkey_disabled)) return; - /* - * TODO: Just set UAMOR to zero if @new_thread hasn't used any keys yet. - */ if (old_thread->amr != new_thread->amr) write_amr(new_thread->amr); if (old_thread->iamr != new_thread->iamr) @@ -305,9 +296,13 @@ void thread_pkey_regs_init(struct thread_struct *thread) if (static_branch_likely(&pkey_disabled)) return; - thread->amr = read_amr() & pkey_amr_uamor_mask; - thread->iamr = read_iamr() & pkey_iamr_mask; - thread->uamor = read_uamor() & pkey_amr_uamor_mask; + thread->amr = pkey_amr_mask; + thread->iamr = pkey_iamr_mask; + thread->uamor = pkey_uamor_mask; + + write_uamor(pkey_uamor_mask); + write_amr(pkey_amr_mask); + write_iamr(pkey_iamr_mask); } static inline bool pkey_allows_readwrite(int pkey) @@ -322,48 +317,7 @@ static inline bool pkey_allows_readwrite(int pkey) int __execute_only_pkey(struct mm_struct *mm) { - bool need_to_set_mm_pkey = false; - int execute_only_pkey = mm->context.execute_only_pkey; - int ret; - - /* Do we need to assign a pkey for mm's execute-only maps? */ - if (execute_only_pkey == -1) { - /* Go allocate one to use, which might fail */ - execute_only_pkey = mm_pkey_alloc(mm); - if (execute_only_pkey < 0) - return -1; - need_to_set_mm_pkey = true; - } - - /* - * We do not want to go through the relatively costly dance to set AMR - * if we do not need to. Check it first and assume that if the - * execute-only pkey is readwrite-disabled than we do not have to set it - * ourselves. - */ - if (!need_to_set_mm_pkey && !pkey_allows_readwrite(execute_only_pkey)) - return execute_only_pkey; - - /* - * Set up AMR so that it denies access for everything other than - * execution. - */ - ret = __arch_set_user_pkey_access(current, execute_only_pkey, - PKEY_DISABLE_ACCESS | - PKEY_DISABLE_WRITE); - /* - * If the AMR-set operation failed somehow, just return 0 and - * effectively disable execute-only support. - */ - if (ret) { - mm_pkey_free(mm, execute_only_pkey); - return -1; - } - - /* We got one, store it and use it from here on out */ - if (need_to_set_mm_pkey) - mm->context.execute_only_pkey = execute_only_pkey; - return execute_only_pkey; + return mm->context.execute_only_pkey; } static inline bool vma_is_pkey_exec_only(struct vm_area_struct *vma) @@ -407,9 +361,6 @@ static bool pkey_access_permitted(int pkey, bool write, bool execute) int pkey_shift; u64 amr; - if (!pkey) - return true; - if (!is_pkey_enabled(pkey)) return true; diff --git a/arch/powerpc/mm/slb.c b/arch/powerpc/mm/slb.c index cb796724a6fc..0b095fa54049 100644 --- a/arch/powerpc/mm/slb.c +++ b/arch/powerpc/mm/slb.c @@ -90,6 +90,45 @@ static inline void create_shadowed_slbe(unsigned long ea, int ssize, : "memory" ); } +/* + * Insert bolted entries into SLB (which may not be empty, so don't clear + * slb_cache_ptr). + */ +void __slb_restore_bolted_realmode(void) +{ + struct slb_shadow *p = get_slb_shadow(); + enum slb_index index; + + /* No isync needed because realmode. */ + for (index = 0; index < SLB_NUM_BOLTED; index++) { + asm volatile("slbmte %0,%1" : + : "r" (be64_to_cpu(p->save_area[index].vsid)), + "r" (be64_to_cpu(p->save_area[index].esid))); + } +} + +/* + * Insert the bolted entries into an empty SLB. + * This is not the same as rebolt because the bolted segments are not + * changed, just loaded from the shadow area. + */ +void slb_restore_bolted_realmode(void) +{ + __slb_restore_bolted_realmode(); + get_paca()->slb_cache_ptr = 0; +} + +/* + * This flushes all SLB entries including 0, so it must be realmode. + */ +void slb_flush_all_realmode(void) +{ + /* + * This flushes all SLB entries including 0, so it must be realmode. + */ + asm volatile("slbmte %0,%0; slbia" : : "r" (0)); +} + static void __slb_flush_and_rebolt(void) { /* If you change this make sure you change SLB_NUM_BOLTED diff --git a/arch/powerpc/mm/slb_low.S b/arch/powerpc/mm/slb_low.S index a83fbd2a4a24..4ac5057ad439 100644 --- a/arch/powerpc/mm/slb_low.S +++ b/arch/powerpc/mm/slb_low.S @@ -22,6 +22,7 @@ #include <asm/mmu.h> #include <asm/pgtable.h> #include <asm/firmware.h> +#include <asm/feature-fixups.h> /* * This macro generates asm code to compute the VSID scramble diff --git a/arch/powerpc/mm/subpage-prot.c b/arch/powerpc/mm/subpage-prot.c index 9d16ee251fc0..3327551c8b47 100644 --- a/arch/powerpc/mm/subpage-prot.c +++ b/arch/powerpc/mm/subpage-prot.c @@ -17,7 +17,6 @@ #include <asm/pgtable.h> #include <linux/uaccess.h> -#include <asm/tlbflush.h> /* * Free all pages allocated for subpage protection maps and pointers. diff --git a/arch/powerpc/mm/tlb-radix.c b/arch/powerpc/mm/tlb-radix.c index 1135b43a597c..fef3e1eb3a19 100644 --- a/arch/powerpc/mm/tlb-radix.c +++ b/arch/powerpc/mm/tlb-radix.c @@ -1048,24 +1048,6 @@ void radix__flush_tlb_all(void) asm volatile("eieio; tlbsync; ptesync": : :"memory"); } -void radix__flush_tlb_pte_p9_dd1(unsigned long old_pte, struct mm_struct *mm, - unsigned long address) -{ - /* - * We track page size in pte only for DD1, So we can - * call this only on DD1. - */ - if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) { - VM_WARN_ON(1); - return; - } - - if (old_pte & R_PAGE_LARGE) - radix__flush_tlb_page_psize(mm, address, MMU_PAGE_2M); - else - radix__flush_tlb_page_psize(mm, address, mmu_virtual_psize); -} - #ifdef CONFIG_KVM_BOOK3S_HV_POSSIBLE extern void radix_kvm_prefetch_workaround(struct mm_struct *mm) { diff --git a/arch/powerpc/mm/tlb_low_64e.S b/arch/powerpc/mm/tlb_low_64e.S index eb82d787d99a..7fd20c52a8ec 100644 --- a/arch/powerpc/mm/tlb_low_64e.S +++ b/arch/powerpc/mm/tlb_low_64e.S @@ -22,6 +22,7 @@ #include <asm/ppc-opcode.h> #include <asm/kvm_asm.h> #include <asm/kvm_booke_hv_asm.h> +#include <asm/feature-fixups.h> #ifdef CONFIG_PPC_64K_PAGES #define VPTE_PMD_SHIFT (PTE_INDEX_SIZE+1) diff --git a/arch/powerpc/mm/tlb_nohash_low.S b/arch/powerpc/mm/tlb_nohash_low.S index 048b8e9f4492..e066a658acac 100644 --- a/arch/powerpc/mm/tlb_nohash_low.S +++ b/arch/powerpc/mm/tlb_nohash_low.S @@ -34,6 +34,8 @@ #include <asm/asm-offsets.h> #include <asm/processor.h> #include <asm/bug.h> +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> #if defined(CONFIG_40x) diff --git a/arch/powerpc/net/Makefile b/arch/powerpc/net/Makefile index 809f019d3cba..c2dec3a68d4c 100644 --- a/arch/powerpc/net/Makefile +++ b/arch/powerpc/net/Makefile @@ -2,7 +2,7 @@ # # Arch-specific network modules # -ifeq ($(CONFIG_PPC64),y) +ifdef CONFIG_PPC64 obj-$(CONFIG_BPF_JIT) += bpf_jit_comp64.o else obj-$(CONFIG_BPF_JIT) += bpf_jit_asm.o bpf_jit_comp.o diff --git a/arch/powerpc/net/bpf_jit32.h b/arch/powerpc/net/bpf_jit32.h index a8cd7e289ecd..6f4daacad296 100644 --- a/arch/powerpc/net/bpf_jit32.h +++ b/arch/powerpc/net/bpf_jit32.h @@ -13,6 +13,7 @@ #ifndef _BPF_JIT32_H #define _BPF_JIT32_H +#include <asm/asm-compat.h> #include "bpf_jit.h" #ifdef CONFIG_PPC64 diff --git a/arch/powerpc/net/bpf_jit_asm.S b/arch/powerpc/net/bpf_jit_asm.S index 3dd9c43d40c9..c80280dc2e04 100644 --- a/arch/powerpc/net/bpf_jit_asm.S +++ b/arch/powerpc/net/bpf_jit_asm.S @@ -10,6 +10,7 @@ */ #include <asm/ppc_asm.h> +#include <asm/asm-compat.h> #include "bpf_jit32.h" /* diff --git a/arch/powerpc/net/bpf_jit_comp.c b/arch/powerpc/net/bpf_jit_comp.c index 5b061fc81df3..d5bfe24bb3b5 100644 --- a/arch/powerpc/net/bpf_jit_comp.c +++ b/arch/powerpc/net/bpf_jit_comp.c @@ -12,6 +12,7 @@ */ #include <linux/moduleloader.h> #include <asm/cacheflush.h> +#include <asm/asm-compat.h> #include <linux/netdevice.h> #include <linux/filter.h> #include <linux/if_vlan.h> diff --git a/arch/powerpc/net/bpf_jit_comp64.c b/arch/powerpc/net/bpf_jit_comp64.c index c0a9bcd28356..50b129785aee 100644 --- a/arch/powerpc/net/bpf_jit_comp64.c +++ b/arch/powerpc/net/bpf_jit_comp64.c @@ -13,6 +13,7 @@ */ #include <linux/moduleloader.h> #include <asm/cacheflush.h> +#include <asm/asm-compat.h> #include <linux/netdevice.h> #include <linux/filter.h> #include <linux/if_vlan.h> diff --git a/arch/powerpc/perf/core-book3s.c b/arch/powerpc/perf/core-book3s.c index 19d8ab49d1bd..81f8a0c838ae 100644 --- a/arch/powerpc/perf/core-book3s.c +++ b/arch/powerpc/perf/core-book3s.c @@ -128,10 +128,6 @@ static inline void power_pmu_bhrb_disable(struct perf_event *event) {} static void power_pmu_sched_task(struct perf_event_context *ctx, bool sched_in) {} static inline void power_pmu_bhrb_read(struct cpu_hw_events *cpuhw) {} static void pmao_restore_workaround(bool ebb) { } -static bool use_ic(u64 event) -{ - return false; -} #endif /* CONFIG_PPC32 */ static bool regs_use_siar(struct pt_regs *regs) @@ -714,14 +710,6 @@ static void pmao_restore_workaround(bool ebb) mtspr(SPRN_PMC6, pmcs[5]); } -static bool use_ic(u64 event) -{ - if (cpu_has_feature(CPU_FTR_POWER9_DD1) && - (event == 0x200f2 || event == 0x300f2)) - return true; - - return false; -} #endif /* CONFIG_PPC64 */ static void perf_event_interrupt(struct pt_regs *regs); @@ -1046,7 +1034,6 @@ static u64 check_and_compute_delta(u64 prev, u64 val) static void power_pmu_read(struct perf_event *event) { s64 val, delta, prev; - struct cpu_hw_events *cpuhw = this_cpu_ptr(&cpu_hw_events); if (event->hw.state & PERF_HES_STOPPED) return; @@ -1056,13 +1043,6 @@ static void power_pmu_read(struct perf_event *event) if (is_ebb_event(event)) { val = read_pmc(event->hw.idx); - if (use_ic(event->attr.config)) { - val = mfspr(SPRN_IC); - if (val > cpuhw->ic_init) - val = val - cpuhw->ic_init; - else - val = val + (0 - cpuhw->ic_init); - } local64_set(&event->hw.prev_count, val); return; } @@ -1076,13 +1056,6 @@ static void power_pmu_read(struct perf_event *event) prev = local64_read(&event->hw.prev_count); barrier(); val = read_pmc(event->hw.idx); - if (use_ic(event->attr.config)) { - val = mfspr(SPRN_IC); - if (val > cpuhw->ic_init) - val = val - cpuhw->ic_init; - else - val = val + (0 - cpuhw->ic_init); - } delta = check_and_compute_delta(prev, val); if (!delta) return; @@ -1535,13 +1508,6 @@ nocheck: event->attr.branch_sample_type); } - /* - * Workaround for POWER9 DD1 to use the Instruction Counter - * register value for instruction counting - */ - if (use_ic(event->attr.config)) - cpuhw->ic_init = mfspr(SPRN_IC); - perf_pmu_enable(event->pmu); local_irq_restore(flags); return ret; diff --git a/arch/powerpc/perf/imc-pmu.c b/arch/powerpc/perf/imc-pmu.c index d1977b61f827..1fafc32b12a0 100644 --- a/arch/powerpc/perf/imc-pmu.c +++ b/arch/powerpc/perf/imc-pmu.c @@ -867,59 +867,6 @@ static int thread_imc_cpu_init(void) ppc_thread_imc_cpu_offline); } -void thread_imc_pmu_sched_task(struct perf_event_context *ctx, - bool sched_in) -{ - int core_id; - struct imc_pmu_ref *ref; - - if (!is_core_imc_mem_inited(smp_processor_id())) - return; - - core_id = smp_processor_id() / threads_per_core; - /* - * imc pmus are enabled only when it is used. - * See if this is triggered for the first time. - * If yes, take the mutex lock and enable the counters. - * If not, just increment the count in ref count struct. - */ - ref = &core_imc_refc[core_id]; - if (!ref) - return; - - if (sched_in) { - mutex_lock(&ref->lock); - if (ref->refc == 0) { - if (opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE, - get_hard_smp_processor_id(smp_processor_id()))) { - mutex_unlock(&ref->lock); - pr_err("thread-imc: Unable to start the counter\ - for core %d\n", core_id); - return; - } - } - ++ref->refc; - mutex_unlock(&ref->lock); - } else { - mutex_lock(&ref->lock); - ref->refc--; - if (ref->refc == 0) { - if (opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE, - get_hard_smp_processor_id(smp_processor_id()))) { - mutex_unlock(&ref->lock); - pr_err("thread-imc: Unable to stop the counters\ - for core %d\n", core_id); - return; - } - } else if (ref->refc < 0) { - ref->refc = 0; - } - mutex_unlock(&ref->lock); - } - - return; -} - static int thread_imc_event_init(struct perf_event *event) { u32 config = event->attr.config; @@ -1046,22 +993,70 @@ static int imc_event_add(struct perf_event *event, int flags) static int thread_imc_event_add(struct perf_event *event, int flags) { + int core_id; + struct imc_pmu_ref *ref; + if (flags & PERF_EF_START) imc_event_start(event, flags); - /* Enable the sched_task to start the engine */ - perf_sched_cb_inc(event->ctx->pmu); + if (!is_core_imc_mem_inited(smp_processor_id())) + return -EINVAL; + + core_id = smp_processor_id() / threads_per_core; + /* + * imc pmus are enabled only when it is used. + * See if this is triggered for the first time. + * If yes, take the mutex lock and enable the counters. + * If not, just increment the count in ref count struct. + */ + ref = &core_imc_refc[core_id]; + if (!ref) + return -EINVAL; + + mutex_lock(&ref->lock); + if (ref->refc == 0) { + if (opal_imc_counters_start(OPAL_IMC_COUNTERS_CORE, + get_hard_smp_processor_id(smp_processor_id()))) { + mutex_unlock(&ref->lock); + pr_err("thread-imc: Unable to start the counter\ + for core %d\n", core_id); + return -EINVAL; + } + } + ++ref->refc; + mutex_unlock(&ref->lock); return 0; } static void thread_imc_event_del(struct perf_event *event, int flags) { + + int core_id; + struct imc_pmu_ref *ref; + /* * Take a snapshot and calculate the delta and update * the event counter values. */ imc_event_update(event); - perf_sched_cb_dec(event->ctx->pmu); + + core_id = smp_processor_id() / threads_per_core; + ref = &core_imc_refc[core_id]; + + mutex_lock(&ref->lock); + ref->refc--; + if (ref->refc == 0) { + if (opal_imc_counters_stop(OPAL_IMC_COUNTERS_CORE, + get_hard_smp_processor_id(smp_processor_id()))) { + mutex_unlock(&ref->lock); + pr_err("thread-imc: Unable to stop the counters\ + for core %d\n", core_id); + return; + } + } else if (ref->refc < 0) { + ref->refc = 0; + } + mutex_unlock(&ref->lock); } /* update_pmu_ops : Populate the appropriate operations for "pmu" */ @@ -1087,7 +1082,6 @@ static int update_pmu_ops(struct imc_pmu *pmu) break; case IMC_DOMAIN_THREAD: pmu->pmu.event_init = thread_imc_event_init; - pmu->pmu.sched_task = thread_imc_pmu_sched_task; pmu->pmu.add = thread_imc_event_add; pmu->pmu.del = thread_imc_event_del; pmu->pmu.start_txn = thread_imc_pmu_start_txn; diff --git a/arch/powerpc/perf/isa207-common.c b/arch/powerpc/perf/isa207-common.c index 2efee3f196f5..177de814286f 100644 --- a/arch/powerpc/perf/isa207-common.c +++ b/arch/powerpc/perf/isa207-common.c @@ -59,7 +59,7 @@ static bool is_event_valid(u64 event) { u64 valid_mask = EVENT_VALID_MASK; - if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1)) + if (cpu_has_feature(CPU_FTR_ARCH_300)) valid_mask = p9_EVENT_VALID_MASK; return !(event & ~valid_mask); @@ -86,8 +86,6 @@ static void mmcra_sdar_mode(u64 event, unsigned long *mmcra) * Incase of Power9: * Marked event: MMCRA[SDAR_MODE] will be set to 0b00 ('No Updates'), * or if group already have any marked events. - * Non-Marked events (for DD1): - * MMCRA[SDAR_MODE] will be set to 0b01 * For rest * MMCRA[SDAR_MODE] will be set from event code. * If sdar_mode from event is zero, default to 0b01. Hardware @@ -96,7 +94,7 @@ static void mmcra_sdar_mode(u64 event, unsigned long *mmcra) if (cpu_has_feature(CPU_FTR_ARCH_300)) { if (is_event_marked(event) || (*mmcra & MMCRA_SAMPLE_ENABLE)) *mmcra &= MMCRA_SDAR_MODE_NO_UPDATES; - else if (!cpu_has_feature(CPU_FTR_POWER9_DD1) && p9_SDAR_MODE(event)) + else if (p9_SDAR_MODE(event)) *mmcra |= p9_SDAR_MODE(event) << MMCRA_SDAR_MODE_SHIFT; else *mmcra |= MMCRA_SDAR_MODE_DCACHE; @@ -106,7 +104,7 @@ static void mmcra_sdar_mode(u64 event, unsigned long *mmcra) static u64 thresh_cmp_val(u64 value) { - if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1)) + if (cpu_has_feature(CPU_FTR_ARCH_300)) return value << p9_MMCRA_THR_CMP_SHIFT; return value << MMCRA_THR_CMP_SHIFT; @@ -114,7 +112,7 @@ static u64 thresh_cmp_val(u64 value) static unsigned long combine_from_event(u64 event) { - if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1)) + if (cpu_has_feature(CPU_FTR_ARCH_300)) return p9_EVENT_COMBINE(event); return EVENT_COMBINE(event); @@ -122,7 +120,7 @@ static unsigned long combine_from_event(u64 event) static unsigned long combine_shift(unsigned long pmc) { - if (cpu_has_feature(CPU_FTR_ARCH_300) && !cpu_has_feature(CPU_FTR_POWER9_DD1)) + if (cpu_has_feature(CPU_FTR_ARCH_300)) return p9_MMCR1_COMBINE_SHIFT(pmc); return MMCR1_COMBINE_SHIFT(pmc); diff --git a/arch/powerpc/perf/isa207-common.h b/arch/powerpc/perf/isa207-common.h index 6a0b586c935a..0028f4b9490d 100644 --- a/arch/powerpc/perf/isa207-common.h +++ b/arch/powerpc/perf/isa207-common.h @@ -158,11 +158,6 @@ CNST_PMC_VAL(1) | CNST_PMC_VAL(2) | CNST_PMC_VAL(3) | \ CNST_PMC_VAL(4) | CNST_PMC_VAL(5) | CNST_PMC_VAL(6) | CNST_NC_VAL -/* - * Lets restrict use of PMC5 for instruction counting. - */ -#define P9_DD1_TEST_ADDER (ISA207_TEST_ADDER | CNST_PMC_VAL(5)) - /* Bits in MMCR1 for PowerISA v2.07 */ #define MMCR1_UNIT_SHIFT(pmc) (60 - (4 * ((pmc) - 1))) #define MMCR1_COMBINE_SHIFT(pmc) (35 - ((pmc) - 1)) diff --git a/arch/powerpc/perf/power9-pmu.c b/arch/powerpc/perf/power9-pmu.c index 2ca0b33b4efb..e012b1030a5b 100644 --- a/arch/powerpc/perf/power9-pmu.c +++ b/arch/powerpc/perf/power9-pmu.c @@ -219,12 +219,6 @@ static struct attribute_group power9_pmu_events_group = { .attrs = power9_events_attr, }; -static const struct attribute_group *power9_isa207_pmu_attr_groups[] = { - &isa207_pmu_format_group, - &power9_pmu_events_group, - NULL, -}; - PMU_FORMAT_ATTR(event, "config:0-51"); PMU_FORMAT_ATTR(pmcxsel, "config:0-7"); PMU_FORMAT_ATTR(mark, "config:8"); @@ -267,17 +261,6 @@ static const struct attribute_group *power9_pmu_attr_groups[] = { NULL, }; -static int power9_generic_events_dd1[] = { - [PERF_COUNT_HW_CPU_CYCLES] = PM_CYC, - [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = PM_ICT_NOSLOT_CYC, - [PERF_COUNT_HW_STALLED_CYCLES_BACKEND] = PM_CMPLU_STALL, - [PERF_COUNT_HW_INSTRUCTIONS] = PM_INST_DISP, - [PERF_COUNT_HW_BRANCH_INSTRUCTIONS] = PM_BR_CMPL_ALT, - [PERF_COUNT_HW_BRANCH_MISSES] = PM_BR_MPRED_CMPL, - [PERF_COUNT_HW_CACHE_REFERENCES] = PM_LD_REF_L1, - [PERF_COUNT_HW_CACHE_MISSES] = PM_LD_MISS_L1_FIN, -}; - static int power9_generic_events[] = { [PERF_COUNT_HW_CPU_CYCLES] = PM_CYC, [PERF_COUNT_HW_STALLED_CYCLES_FRONTEND] = PM_ICT_NOSLOT_CYC, @@ -439,25 +422,6 @@ static int power9_cache_events[C(MAX)][C(OP_MAX)][C(RESULT_MAX)] = { #undef C -static struct power_pmu power9_isa207_pmu = { - .name = "POWER9", - .n_counter = MAX_PMU_COUNTERS, - .add_fields = ISA207_ADD_FIELDS, - .test_adder = P9_DD1_TEST_ADDER, - .compute_mmcr = isa207_compute_mmcr, - .config_bhrb = power9_config_bhrb, - .bhrb_filter_map = power9_bhrb_filter_map, - .get_constraint = isa207_get_constraint, - .get_alternatives = power9_get_alternatives, - .disable_pmc = isa207_disable_pmc, - .flags = PPMU_NO_SIAR | PPMU_ARCH_207S, - .n_generic = ARRAY_SIZE(power9_generic_events_dd1), - .generic_events = power9_generic_events_dd1, - .cache_events = &power9_cache_events, - .attr_groups = power9_isa207_pmu_attr_groups, - .bhrb_nr = 32, -}; - static struct power_pmu power9_pmu = { .name = "POWER9", .n_counter = MAX_PMU_COUNTERS, @@ -500,23 +464,7 @@ static int __init init_power9_pmu(void) } } - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { - /* - * Since PM_INST_CMPL may not provide right counts in all - * sampling scenarios in power9 DD1, instead use PM_INST_DISP. - */ - EVENT_VAR(PM_INST_CMPL, _g).id = PM_INST_DISP; - /* - * Power9 DD1 should use PM_BR_CMPL_ALT event code for - * "branches" to provide correct counter value. - */ - EVENT_VAR(PM_BR_CMPL, _g).id = PM_BR_CMPL_ALT; - EVENT_VAR(PM_BR_CMPL, _c).id = PM_BR_CMPL_ALT; - rc = register_power_pmu(&power9_isa207_pmu); - } else { - rc = register_power_pmu(&power9_pmu); - } - + rc = register_power_pmu(&power9_pmu); if (rc) return rc; diff --git a/arch/powerpc/perf/req-gen/_begin.h b/arch/powerpc/perf/req-gen/_begin.h index 549f8782c52d..a200b86eba3b 100644 --- a/arch/powerpc/perf/req-gen/_begin.h +++ b/arch/powerpc/perf/req-gen/_begin.h @@ -3,6 +3,8 @@ #ifndef POWERPC_PERF_REQ_GEN_H_ #define POWERPC_PERF_REQ_GEN_H_ +#include <linux/stringify.h> + #define CAT2_STR_(t, s) __stringify(t/s) #define CAT2_STR(t, s) CAT2_STR_(t, s) #define I(...) __VA_ARGS__ diff --git a/arch/powerpc/perf/req-gen/perf.h b/arch/powerpc/perf/req-gen/perf.h index 871a9a1766c2..fa9bc804e67a 100644 --- a/arch/powerpc/perf/req-gen/perf.h +++ b/arch/powerpc/perf/req-gen/perf.h @@ -3,6 +3,7 @@ #define LINUX_POWERPC_PERF_REQ_GEN_PERF_H_ #include <linux/perf_event.h> +#include <linux/stringify.h> #ifndef REQUEST_FILE #error "REQUEST_FILE must be defined before including" diff --git a/arch/powerpc/platforms/4xx/msi.c b/arch/powerpc/platforms/4xx/msi.c index 81b2cbce7df8..7c324eff2f22 100644 --- a/arch/powerpc/platforms/4xx/msi.c +++ b/arch/powerpc/platforms/4xx/msi.c @@ -146,13 +146,19 @@ static int ppc4xx_setup_pcieh_hw(struct platform_device *dev, const u32 *sdr_addr; dma_addr_t msi_phys; void *msi_virt; + int err; sdr_addr = of_get_property(dev->dev.of_node, "sdr-base", NULL); if (!sdr_addr) - return -1; + return -EINVAL; - mtdcri(SDR0, *sdr_addr, upper_32_bits(res.start)); /*HIGH addr */ - mtdcri(SDR0, *sdr_addr + 1, lower_32_bits(res.start)); /* Low addr */ + msi_data = of_get_property(dev->dev.of_node, "msi-data", NULL); + if (!msi_data) + return -EINVAL; + + msi_mask = of_get_property(dev->dev.of_node, "msi-mask", NULL); + if (!msi_mask) + return -EINVAL; msi->msi_dev = of_find_node_by_name(NULL, "ppc4xx-msi"); if (!msi->msi_dev) @@ -160,30 +166,30 @@ static int ppc4xx_setup_pcieh_hw(struct platform_device *dev, msi->msi_regs = of_iomap(msi->msi_dev, 0); if (!msi->msi_regs) { - dev_err(&dev->dev, "of_iomap problem failed\n"); - return -ENOMEM; + dev_err(&dev->dev, "of_iomap failed\n"); + err = -ENOMEM; + goto node_put; } dev_dbg(&dev->dev, "PCIE-MSI: msi register mapped 0x%x 0x%x\n", (u32) (msi->msi_regs + PEIH_TERMADH), (u32) (msi->msi_regs)); msi_virt = dma_alloc_coherent(&dev->dev, 64, &msi_phys, GFP_KERNEL); - if (!msi_virt) - return -ENOMEM; + if (!msi_virt) { + err = -ENOMEM; + goto iounmap; + } msi->msi_addr_hi = upper_32_bits(msi_phys); msi->msi_addr_lo = lower_32_bits(msi_phys & 0xffffffff); dev_dbg(&dev->dev, "PCIE-MSI: msi address high 0x%x, low 0x%x\n", msi->msi_addr_hi, msi->msi_addr_lo); + mtdcri(SDR0, *sdr_addr, upper_32_bits(res.start)); /*HIGH addr */ + mtdcri(SDR0, *sdr_addr + 1, lower_32_bits(res.start)); /* Low addr */ + /* Progam the Interrupt handler Termination addr registers */ out_be32(msi->msi_regs + PEIH_TERMADH, msi->msi_addr_hi); out_be32(msi->msi_regs + PEIH_TERMADL, msi->msi_addr_lo); - msi_data = of_get_property(dev->dev.of_node, "msi-data", NULL); - if (!msi_data) - return -1; - msi_mask = of_get_property(dev->dev.of_node, "msi-mask", NULL); - if (!msi_mask) - return -1; /* Program MSI Expected data and Mask bits */ out_be32(msi->msi_regs + PEIH_MSIED, *msi_data); out_be32(msi->msi_regs + PEIH_MSIMK, *msi_mask); @@ -191,6 +197,12 @@ static int ppc4xx_setup_pcieh_hw(struct platform_device *dev, dma_free_coherent(&dev->dev, 64, msi_virt, msi_phys); return 0; + +iounmap: + iounmap(msi->msi_regs); +node_put: + of_node_put(msi->msi_dev); + return err; } static int ppc4xx_of_msi_remove(struct platform_device *dev) @@ -209,7 +221,6 @@ static int ppc4xx_of_msi_remove(struct platform_device *dev) msi_bitmap_free(&msi->bitmap); iounmap(msi->msi_regs); of_node_put(msi->msi_dev); - kfree(msi); return 0; } @@ -223,18 +234,16 @@ static int ppc4xx_msi_probe(struct platform_device *dev) dev_dbg(&dev->dev, "PCIE-MSI: Setting up MSI support...\n"); - msi = kzalloc(sizeof(*msi), GFP_KERNEL); - if (!msi) { - dev_err(&dev->dev, "No memory for MSI structure\n"); + msi = devm_kzalloc(&dev->dev, sizeof(*msi), GFP_KERNEL); + if (!msi) return -ENOMEM; - } dev->dev.platform_data = msi; /* Get MSI ranges */ err = of_address_to_resource(dev->dev.of_node, 0, &res); if (err) { dev_err(&dev->dev, "%pOF resource error!\n", dev->dev.of_node); - goto error_out; + return err; } msi_irqs = of_irq_count(dev->dev.of_node); @@ -243,7 +252,7 @@ static int ppc4xx_msi_probe(struct platform_device *dev) err = ppc4xx_setup_pcieh_hw(dev, res, msi); if (err) - goto error_out; + return err; err = ppc4xx_msi_init_allocator(dev, msi); if (err) { @@ -256,7 +265,7 @@ static int ppc4xx_msi_probe(struct platform_device *dev) phb->controller_ops.setup_msi_irqs = ppc4xx_setup_msi_irqs; phb->controller_ops.teardown_msi_irqs = ppc4xx_teardown_msi_irqs; } - return err; + return 0; error_out: ppc4xx_of_msi_remove(dev); diff --git a/arch/powerpc/platforms/52xx/Makefile b/arch/powerpc/platforms/52xx/Makefile index ff2f86fe5429..f40d48eab779 100644 --- a/arch/powerpc/platforms/52xx/Makefile +++ b/arch/powerpc/platforms/52xx/Makefile @@ -11,7 +11,7 @@ obj-$(CONFIG_PPC_LITE5200) += lite5200.o obj-$(CONFIG_PPC_MEDIA5200) += media5200.o obj-$(CONFIG_PM) += mpc52xx_sleep.o mpc52xx_pm.o -ifeq ($(CONFIG_PPC_LITE5200),y) +ifdef CONFIG_PPC_LITE5200 obj-$(CONFIG_PM) += lite5200_sleep.o lite5200_pm.o endif diff --git a/arch/powerpc/platforms/52xx/mpc52xx_pm.c b/arch/powerpc/platforms/52xx/mpc52xx_pm.c index 31d3515672f3..b1d208ded981 100644 --- a/arch/powerpc/platforms/52xx/mpc52xx_pm.c +++ b/arch/powerpc/platforms/52xx/mpc52xx_pm.c @@ -117,7 +117,10 @@ int mpc52xx_pm_enter(suspend_state_t state) u32 intr_main_mask; void __iomem * irq_0x500 = (void __iomem *)CONFIG_KERNEL_START + 0x500; unsigned long irq_0x500_stop = (unsigned long)irq_0x500 + mpc52xx_ds_cached_size; - char saved_0x500[mpc52xx_ds_cached_size]; + char saved_0x500[0x600-0x500]; + + if (WARN_ON(mpc52xx_ds_cached_size > sizeof(saved_0x500))) + return -ENOMEM; /* disable all interrupts in PIC */ intr_main_mask = in_be32(&intr->main_mask); diff --git a/arch/powerpc/platforms/85xx/t1042rdb_diu.c b/arch/powerpc/platforms/85xx/t1042rdb_diu.c index 58fa3d319f1c..dac36ba82fea 100644 --- a/arch/powerpc/platforms/85xx/t1042rdb_diu.c +++ b/arch/powerpc/platforms/85xx/t1042rdb_diu.c @@ -9,8 +9,10 @@ * option) any later version. */ +#include <linux/init.h> #include <linux/io.h> #include <linux/kernel.h> +#include <linux/module.h> #include <linux/of.h> #include <linux/of_address.h> @@ -150,3 +152,5 @@ static int __init t1042rdb_diu_init(void) } early_initcall(t1042rdb_diu_init); + +MODULE_LICENSE("GPL"); diff --git a/arch/powerpc/platforms/Kconfig.cputype b/arch/powerpc/platforms/Kconfig.cputype index e6a1de521319..6c6a7c72cae4 100644 --- a/arch/powerpc/platforms/Kconfig.cputype +++ b/arch/powerpc/platforms/Kconfig.cputype @@ -74,7 +74,6 @@ config PPC_BOOK3S_64 select HAVE_ARCH_TRANSPARENT_HUGEPAGE select ARCH_SUPPORTS_NUMA_BALANCING select IRQ_WORK - select HAVE_KERNEL_XZ config PPC_BOOK3E_64 bool "Embedded processors" @@ -86,7 +85,6 @@ endchoice choice prompt "CPU selection" - depends on PPC64 default GENERIC_CPU help This will create a kernel which is optimised for a particular CPU. @@ -96,13 +94,17 @@ choice config GENERIC_CPU bool "Generic (POWER4 and above)" - depends on !CPU_LITTLE_ENDIAN + depends on PPC64 && !CPU_LITTLE_ENDIAN config GENERIC_CPU bool "Generic (POWER8 and above)" - depends on CPU_LITTLE_ENDIAN + depends on PPC64 && CPU_LITTLE_ENDIAN select ARCH_HAS_FAST_MULTIPLIER +config GENERIC_CPU + bool "Generic 32 bits powerpc" + depends on PPC32 && !PPC_8xx + config CELL_CPU bool "Cell Broadband Engine" depends on PPC_BOOK3S_64 && !CPU_LITTLE_ENDIAN @@ -138,8 +140,37 @@ config E6500_CPU bool "Freescale e6500" depends on E500 +config 860_CPU + bool "8xx family" + depends on PPC_8xx + +config E300C2_CPU + bool "e300c2 (832x)" + depends on PPC_BOOK3S_32 + +config E300C3_CPU + bool "e300c3 (831x)" + depends on PPC_BOOK3S_32 + endchoice +config TARGET_CPU_BOOL + bool + default !GENERIC_CPU + +config TARGET_CPU + string + depends on TARGET_CPU_BOOL + default "cell" if CELL_CPU + default "power5" if POWER5_CPU + default "power6" if POWER6_CPU + default "power7" if POWER7_CPU + default "power8" if POWER8_CPU + default "power9" if POWER9_CPU + default "860" if 860_CPU + default "e300c2" if E300C2_CPU + default "e300c3" if E300C3_CPU + config PPC_BOOK3S def_bool y depends on PPC_BOOK3S_32 || PPC_BOOK3S_64 diff --git a/arch/powerpc/platforms/cell/Makefile b/arch/powerpc/platforms/cell/Makefile index d5f808e8a5f3..10064a33ca96 100644 --- a/arch/powerpc/platforms/cell/Makefile +++ b/arch/powerpc/platforms/cell/Makefile @@ -10,7 +10,7 @@ obj-$(CONFIG_CBE_CPUFREQ_SPU_GOVERNOR) += cpufreq_spudemand.o obj-$(CONFIG_PPC_IBM_CELL_POWERBUTTON) += cbe_powerbutton.o -ifeq ($(CONFIG_SMP),y) +ifdef CONFIG_SMP obj-$(CONFIG_PPC_CELL_NATIVE) += smp.o endif diff --git a/arch/powerpc/platforms/cell/cbe_thermal.c b/arch/powerpc/platforms/cell/cbe_thermal.c index 2c15ff094483..55aac74e1cb9 100644 --- a/arch/powerpc/platforms/cell/cbe_thermal.c +++ b/arch/powerpc/platforms/cell/cbe_thermal.c @@ -49,6 +49,7 @@ #include <linux/device.h> #include <linux/kernel.h> #include <linux/cpu.h> +#include <linux/stringify.h> #include <asm/spu.h> #include <asm/io.h> #include <asm/prom.h> diff --git a/arch/powerpc/platforms/cell/spufs/sputrace.h b/arch/powerpc/platforms/cell/spufs/sputrace.h index d557e999b662..1def11e911ac 100644 --- a/arch/powerpc/platforms/cell/spufs/sputrace.h +++ b/arch/powerpc/platforms/cell/spufs/sputrace.h @@ -3,6 +3,7 @@ #define _TRACE_SPUFS_H #include <linux/tracepoint.h> +#include <linux/stringify.h> #undef TRACE_SYSTEM #define TRACE_SYSTEM spufs diff --git a/arch/powerpc/platforms/chrp/nvram.c b/arch/powerpc/platforms/chrp/nvram.c index c3ede2c365c3..791b86398e1d 100644 --- a/arch/powerpc/platforms/chrp/nvram.c +++ b/arch/powerpc/platforms/chrp/nvram.c @@ -11,6 +11,7 @@ */ #include <linux/kernel.h> +#include <linux/module.h> #include <linux/init.h> #include <linux/spinlock.h> #include <linux/uaccess.h> @@ -89,3 +90,5 @@ void __init chrp_nvram_init(void) return; } + +MODULE_LICENSE("GPL v2"); diff --git a/arch/powerpc/platforms/embedded6xx/wii.c b/arch/powerpc/platforms/embedded6xx/wii.c index 8bb46dcbebd8..403523c061ba 100644 --- a/arch/powerpc/platforms/embedded6xx/wii.c +++ b/arch/powerpc/platforms/embedded6xx/wii.c @@ -67,16 +67,6 @@ void __init wii_memory_fixups(void) { struct memblock_region *p = memblock.memory.regions; - /* - * This is part of a workaround to allow the use of two - * discontinuous RAM ranges on the Wii, even if this is - * currently unsupported on 32-bit PowerPC Linux. - * - * We coalesce the two memory ranges of the Wii into a - * single range, then create a reservation for the "hole" - * between both ranges. - */ - BUG_ON(memblock.memory.cnt != 2); BUG_ON(!page_aligned(p[0].base) || !page_aligned(p[1].base)); diff --git a/arch/powerpc/platforms/pasemi/dma_lib.c b/arch/powerpc/platforms/pasemi/dma_lib.c index 2c72263ad6ab..c80f72c370ae 100644 --- a/arch/powerpc/platforms/pasemi/dma_lib.c +++ b/arch/powerpc/platforms/pasemi/dma_lib.c @@ -531,7 +531,7 @@ int pasemi_dma_init(void) iob_pdev = pci_get_device(PCI_VENDOR_ID_PASEMI, 0xa001, NULL); if (!iob_pdev) { BUG(); - printk(KERN_WARNING "Can't find I/O Bridge\n"); + pr_warn("Can't find I/O Bridge\n"); err = -ENODEV; goto out; } @@ -540,7 +540,7 @@ int pasemi_dma_init(void) dma_pdev = pci_get_device(PCI_VENDOR_ID_PASEMI, 0xa007, NULL); if (!dma_pdev) { BUG(); - printk(KERN_WARNING "Can't find DMA controller\n"); + pr_warn("Can't find DMA controller\n"); err = -ENODEV; goto out; } @@ -623,7 +623,7 @@ int pasemi_dma_init(void) pasemi_write_dma_reg(PAS_DMA_TXF_CFLG0, 0xffffffff); pasemi_write_dma_reg(PAS_DMA_TXF_CFLG1, 0xffffffff); - printk(KERN_INFO "PA Semi PWRficient DMA library initialized " + pr_info("PA Semi PWRficient DMA library initialized " "(%d tx, %d rx channels)\n", num_txch, num_rxch); out: diff --git a/arch/powerpc/platforms/pasemi/gpio_mdio.c b/arch/powerpc/platforms/pasemi/gpio_mdio.c index c23e60959aa8..6f35a2afe522 100644 --- a/arch/powerpc/platforms/pasemi/gpio_mdio.c +++ b/arch/powerpc/platforms/pasemi/gpio_mdio.c @@ -256,7 +256,7 @@ static int gpio_mdio_probe(struct platform_device *ofdev) err = of_mdiobus_register(new_bus, np); if (err != 0) { - printk(KERN_ERR "%s: Cannot register as MDIO bus, err %d\n", + pr_err("%s: Cannot register as MDIO bus, err %d\n", new_bus->name, err); goto out_free_irq; } diff --git a/arch/powerpc/platforms/pasemi/idle.c b/arch/powerpc/platforms/pasemi/idle.c index 44e0d9226f0a..8bb4e8082441 100644 --- a/arch/powerpc/platforms/pasemi/idle.c +++ b/arch/powerpc/platforms/pasemi/idle.c @@ -78,13 +78,13 @@ static int pasemi_system_reset_exception(struct pt_regs *regs) static int __init pasemi_idle_init(void) { #ifndef CONFIG_PPC_PASEMI_CPUFREQ - printk(KERN_WARNING "No cpufreq driver, powersavings modes disabled\n"); + pr_warn("No cpufreq driver, powersavings modes disabled\n"); current_mode = 0; #endif ppc_md.system_reset_exception = pasemi_system_reset_exception; ppc_md.power_save = modes[current_mode].entry; - printk(KERN_INFO "Using PA6T idle loop (%s)\n", modes[current_mode].name); + pr_info("Using PA6T idle loop (%s)\n", modes[current_mode].name); return 0; } diff --git a/arch/powerpc/platforms/pasemi/iommu.c b/arch/powerpc/platforms/pasemi/iommu.c index 78b80cbd9768..f06c83f321e6 100644 --- a/arch/powerpc/platforms/pasemi/iommu.c +++ b/arch/powerpc/platforms/pasemi/iommu.c @@ -210,7 +210,7 @@ static int __init iob_init(struct device_node *dn) /* For 2G space, 8x64 pages (2^21 bytes) is max total l2 size */ iob_l2_base = (u32 *)__va(memblock_alloc_base(1UL<<21, 1UL<<21, 0x80000000)); - printk(KERN_INFO "IOBMAP L2 allocated at: %p\n", iob_l2_base); + pr_info("IOBMAP L2 allocated at: %p\n", iob_l2_base); /* Allocate a spare page to map all invalid IOTLB pages. */ tmp = memblock_alloc(IOBMAP_PAGE_SIZE, IOBMAP_PAGE_SIZE); diff --git a/arch/powerpc/platforms/pasemi/misc.c b/arch/powerpc/platforms/pasemi/misc.c index 8571e7bf78b6..aa958a46957f 100644 --- a/arch/powerpc/platforms/pasemi/misc.c +++ b/arch/powerpc/platforms/pasemi/misc.c @@ -69,9 +69,7 @@ static int __init pasemi_register_i2c_devices(void) addr = of_get_property(node, "reg", &len); if (!addr || len < sizeof(int) || *addr > (1 << 10) - 1) { - printk(KERN_WARNING - "pasemi_register_i2c_devices: " - "invalid i2c device entry\n"); + pr_warn("pasemi_register_i2c_devices: invalid i2c device entry\n"); continue; } diff --git a/arch/powerpc/platforms/pasemi/pci.c b/arch/powerpc/platforms/pasemi/pci.c index aea9ff2c8e6d..c3c64172482d 100644 --- a/arch/powerpc/platforms/pasemi/pci.c +++ b/arch/powerpc/platforms/pasemi/pci.c @@ -205,7 +205,7 @@ static int __init pas_add_bridge(struct device_node *dev) setup_pa_pxp(hose); - printk(KERN_INFO "Found PA-PXP PCI host bridge.\n"); + pr_info("Found PA-PXP PCI host bridge.\n"); /* Interpret the "ranges" property */ pci_process_bridge_OF_ranges(hose, dev, 1); @@ -216,21 +216,21 @@ static int __init pas_add_bridge(struct device_node *dev) void __init pas_pci_init(void) { struct device_node *np, *root; + int res; root = of_find_node_by_path("/"); if (!root) { - printk(KERN_CRIT "pas_pci_init: can't find root " - "of device tree\n"); + pr_crit("pas_pci_init: can't find root of device tree\n"); return; } pci_set_flags(PCI_SCAN_ALL_PCIE_DEVS); - for (np = NULL; (np = of_get_next_child(root, np)) != NULL;) - if (np->name && !strcmp(np->name, "pxp") && !pas_add_bridge(np)) - of_node_get(np); - - of_node_put(root); + np = of_find_compatible_node(root, NULL, "pasemi,rootbus"); + if (np) { + res = pas_add_bridge(np); + of_node_put(np); + } } void __iomem *pasemi_pci_getcfgaddr(struct pci_dev *dev, int offset) diff --git a/arch/powerpc/platforms/pasemi/setup.c b/arch/powerpc/platforms/pasemi/setup.c index d0b8ae53660d..9a6eb04cca83 100644 --- a/arch/powerpc/platforms/pasemi/setup.c +++ b/arch/powerpc/platforms/pasemi/setup.c @@ -207,8 +207,7 @@ static __init void pas_init_IRQ(void) break; } if (!mpic_node) { - printk(KERN_ERR - "Failed to locate the MPIC interrupt controller\n"); + pr_err("Failed to locate the MPIC interrupt controller\n"); return; } @@ -217,12 +216,12 @@ static __init void pas_init_IRQ(void) naddr = of_n_addr_cells(root); opprop = of_get_property(root, "platform-open-pic", &opplen); if (!opprop) { - printk(KERN_ERR "No platform-open-pic property.\n"); + pr_err("No platform-open-pic property.\n"); of_node_put(root); return; } openpic_addr = of_read_number(opprop, naddr); - printk(KERN_DEBUG "OpenPIC addr: %lx\n", openpic_addr); + pr_debug("OpenPIC addr: %lx\n", openpic_addr); mpic_flags = MPIC_LARGE_VECTORS | MPIC_NO_BIAS | MPIC_NO_RESET; @@ -265,72 +264,72 @@ static int pas_machine_check_handler(struct pt_regs *regs) srr1 = regs->msr; if (nmi_virq && mpic_get_mcirq() == nmi_virq) { - printk(KERN_ERR "NMI delivered\n"); + pr_err("NMI delivered\n"); debugger(regs); mpic_end_irq(irq_get_irq_data(nmi_virq)); goto out; } dsisr = mfspr(SPRN_DSISR); - printk(KERN_ERR "Machine Check on CPU %d\n", cpu); - printk(KERN_ERR "SRR0 0x%016lx SRR1 0x%016lx\n", srr0, srr1); - printk(KERN_ERR "DSISR 0x%016lx DAR 0x%016lx\n", dsisr, regs->dar); - printk(KERN_ERR "BER 0x%016lx MER 0x%016lx\n", mfspr(SPRN_PA6T_BER), + pr_err("Machine Check on CPU %d\n", cpu); + pr_err("SRR0 0x%016lx SRR1 0x%016lx\n", srr0, srr1); + pr_err("DSISR 0x%016lx DAR 0x%016lx\n", dsisr, regs->dar); + pr_err("BER 0x%016lx MER 0x%016lx\n", mfspr(SPRN_PA6T_BER), mfspr(SPRN_PA6T_MER)); - printk(KERN_ERR "IER 0x%016lx DER 0x%016lx\n", mfspr(SPRN_PA6T_IER), + pr_err("IER 0x%016lx DER 0x%016lx\n", mfspr(SPRN_PA6T_IER), mfspr(SPRN_PA6T_DER)); - printk(KERN_ERR "Cause:\n"); + pr_err("Cause:\n"); if (srr1 & 0x200000) - printk(KERN_ERR "Signalled by SDC\n"); + pr_err("Signalled by SDC\n"); if (srr1 & 0x100000) { - printk(KERN_ERR "Load/Store detected error:\n"); + pr_err("Load/Store detected error:\n"); if (dsisr & 0x8000) - printk(KERN_ERR "D-cache ECC double-bit error or bus error\n"); + pr_err("D-cache ECC double-bit error or bus error\n"); if (dsisr & 0x4000) - printk(KERN_ERR "LSU snoop response error\n"); + pr_err("LSU snoop response error\n"); if (dsisr & 0x2000) { - printk(KERN_ERR "MMU SLB multi-hit or invalid B field\n"); + pr_err("MMU SLB multi-hit or invalid B field\n"); dump_slb = 1; } if (dsisr & 0x1000) - printk(KERN_ERR "Recoverable Duptags\n"); + pr_err("Recoverable Duptags\n"); if (dsisr & 0x800) - printk(KERN_ERR "Recoverable D-cache parity error count overflow\n"); + pr_err("Recoverable D-cache parity error count overflow\n"); if (dsisr & 0x400) - printk(KERN_ERR "TLB parity error count overflow\n"); + pr_err("TLB parity error count overflow\n"); } if (srr1 & 0x80000) - printk(KERN_ERR "Bus Error\n"); + pr_err("Bus Error\n"); if (srr1 & 0x40000) { - printk(KERN_ERR "I-side SLB multiple hit\n"); + pr_err("I-side SLB multiple hit\n"); dump_slb = 1; } if (srr1 & 0x20000) - printk(KERN_ERR "I-cache parity error hit\n"); + pr_err("I-cache parity error hit\n"); if (num_mce_regs == 0) - printk(KERN_ERR "No MCE registers mapped yet, can't dump\n"); + pr_err("No MCE registers mapped yet, can't dump\n"); else - printk(KERN_ERR "SoC debug registers:\n"); + pr_err("SoC debug registers:\n"); for (i = 0; i < num_mce_regs; i++) - printk(KERN_ERR "%s: 0x%08x\n", mce_regs[i].name, + pr_err("%s: 0x%08x\n", mce_regs[i].name, in_le32(mce_regs[i].addr)); if (dump_slb) { unsigned long e, v; int i; - printk(KERN_ERR "slb contents:\n"); + pr_err("slb contents:\n"); for (i = 0; i < mmu_slb_size; i++) { asm volatile("slbmfee %0,%1" : "=r" (e) : "r" (i)); asm volatile("slbmfev %0,%1" : "=r" (v) : "r" (i)); - printk(KERN_ERR "%02d %016lx %016lx\n", i, e, v); + pr_err("%02d %016lx %016lx\n", i, e, v); } } diff --git a/arch/powerpc/platforms/powermac/cache.S b/arch/powerpc/platforms/powermac/cache.S index cc5347eb1662..27862feee4a5 100644 --- a/arch/powerpc/platforms/powermac/cache.S +++ b/arch/powerpc/platforms/powermac/cache.S @@ -17,6 +17,7 @@ #include <asm/processor.h> #include <asm/ppc_asm.h> #include <asm/cputable.h> +#include <asm/feature-fixups.h> /* * Flush and disable all data caches (dL1, L2, L3). This is used diff --git a/arch/powerpc/platforms/powermac/feature.c b/arch/powerpc/platforms/powermac/feature.c index 3f82cb24eb2b..4eb8cb38fc69 100644 --- a/arch/powerpc/platforms/powermac/feature.c +++ b/arch/powerpc/platforms/powermac/feature.c @@ -2889,10 +2889,8 @@ set_initial_features(void) /* On all machines, switch modem & serial ports off */ for_each_node_by_name(np, "ch-a") initial_serial_shutdown(np); - of_node_put(np); for_each_node_by_name(np, "ch-b") initial_serial_shutdown(np); - of_node_put(np); } void __init diff --git a/arch/powerpc/platforms/powermac/pci.c b/arch/powerpc/platforms/powermac/pci.c index df762bb3c735..04527d13d5a4 100644 --- a/arch/powerpc/platforms/powermac/pci.c +++ b/arch/powerpc/platforms/powermac/pci.c @@ -781,12 +781,12 @@ static int __init pmac_add_bridge(struct device_node *dev) struct resource rsrc; char *disp_name; const int *bus_range; - int primary = 1, has_address = 0; + int primary = 1; DBG("Adding PCI host bridge %pOF\n", dev); /* Fetch host bridge registers address */ - has_address = (of_address_to_resource(dev, 0, &rsrc) == 0); + of_address_to_resource(dev, 0, &rsrc); /* Get bus range if any */ bus_range = of_get_property(dev, "bus-range", &len); @@ -904,7 +904,7 @@ static int pmac_pci_root_bridge_prepare(struct pci_host_bridge *bridge) void __init pmac_pci_init(void) { struct device_node *np, *root; - struct device_node *ht = NULL; + struct device_node *ht __maybe_unused = NULL; pci_set_flags(PCI_CAN_SKIP_ISA_ALIGN); @@ -1019,7 +1019,7 @@ static bool pmac_pci_enable_device_hook(struct pci_dev *dev) return true; } -void pmac_pci_fixup_ohci(struct pci_dev *dev) +static void pmac_pci_fixup_ohci(struct pci_dev *dev) { struct device_node *node = pci_device_to_OF_node(dev); @@ -1054,7 +1054,7 @@ void __init pmac_pcibios_after_init(void) } } -void pmac_pci_fixup_cardbus(struct pci_dev* dev) +static void pmac_pci_fixup_cardbus(struct pci_dev *dev) { if (!machine_is(powermac)) return; @@ -1091,7 +1091,7 @@ void pmac_pci_fixup_cardbus(struct pci_dev* dev) DECLARE_PCI_FIXUP_FINAL(PCI_VENDOR_ID_TI, PCI_ANY_ID, pmac_pci_fixup_cardbus); -void pmac_pci_fixup_pciata(struct pci_dev* dev) +static void pmac_pci_fixup_pciata(struct pci_dev *dev) { u8 progif = 0; diff --git a/arch/powerpc/platforms/powermac/sleep.S b/arch/powerpc/platforms/powermac/sleep.S index 1c2802fabd57..f89808b9713d 100644 --- a/arch/powerpc/platforms/powermac/sleep.S +++ b/arch/powerpc/platforms/powermac/sleep.S @@ -18,6 +18,7 @@ #include <asm/thread_info.h> #include <asm/asm-offsets.h> #include <asm/mmu.h> +#include <asm/feature-fixups.h> #define MAGIC 0x4c617273 /* 'Lars' */ diff --git a/arch/powerpc/platforms/powermac/time.c b/arch/powerpc/platforms/powermac/time.c index 12e6e4d30602..f92c1918fb56 100644 --- a/arch/powerpc/platforms/powermac/time.c +++ b/arch/powerpc/platforms/powermac/time.c @@ -34,6 +34,8 @@ #include <asm/nvram.h> #include <asm/smu.h> +#include "pmac.h" + #undef DEBUG #ifdef DEBUG @@ -249,7 +251,7 @@ int pmac_set_rtc_time(struct rtc_time *tm) * Calibrate the decrementer register using VIA timer 1. * This is used both on powermacs and CHRP machines. */ -int __init via_calibrate_decr(void) +static int __init via_calibrate_decr(void) { struct device_node *vias; volatile unsigned char __iomem *via; diff --git a/arch/powerpc/platforms/powermac/udbg_scc.c b/arch/powerpc/platforms/powermac/udbg_scc.c index d83135a9830e..8901973ed683 100644 --- a/arch/powerpc/platforms/powermac/udbg_scc.c +++ b/arch/powerpc/platforms/powermac/udbg_scc.c @@ -73,7 +73,7 @@ void udbg_scc_init(int force_scc) struct device_node *stdout = NULL, *escc = NULL, *macio = NULL; struct device_node *ch, *ch_def = NULL, *ch_a = NULL; const char *path; - int i, x; + int i; escc = of_find_node_by_name(NULL, "escc"); if (escc == NULL) @@ -120,7 +120,7 @@ void udbg_scc_init(int force_scc) mb(); for (i = 20000; i != 0; --i) - x = in_8(sccc); + in_8(sccc); out_8(sccc, 0x09); /* reset A or B side */ out_8(sccc, 0xc0); diff --git a/arch/powerpc/platforms/powernv/Makefile b/arch/powerpc/platforms/powernv/Makefile index 703a350a7f4e..b540ce8eec55 100644 --- a/arch/powerpc/platforms/powernv/Makefile +++ b/arch/powerpc/platforms/powernv/Makefile @@ -6,7 +6,7 @@ obj-y += opal-msglog.o opal-hmi.o opal-power.o opal-irqchip.o obj-y += opal-kmsg.o opal-powercap.o opal-psr.o opal-sensor-groups.o obj-$(CONFIG_SMP) += smp.o subcore.o subcore-asm.o -obj-$(CONFIG_PCI) += pci.o pci-ioda.o npu-dma.o +obj-$(CONFIG_PCI) += pci.o pci-ioda.o npu-dma.o pci-ioda-tce.o obj-$(CONFIG_CXL_BASE) += pci-cxl.o obj-$(CONFIG_EEH) += eeh-powernv.o obj-$(CONFIG_PPC_SCOM) += opal-xscom.o diff --git a/arch/powerpc/platforms/powernv/eeh-powernv.c b/arch/powerpc/platforms/powernv/eeh-powernv.c index ddfc3544d285..3c1beae29f2d 100644 --- a/arch/powerpc/platforms/powernv/eeh-powernv.c +++ b/arch/powerpc/platforms/powernv/eeh-powernv.c @@ -223,6 +223,14 @@ int pnv_eeh_post_init(void) eeh_probe_devices(); eeh_addr_cache_build(); + if (eeh_has_flag(EEH_POSTPONED_PROBE)) { + eeh_clear_flag(EEH_POSTPONED_PROBE); + if (eeh_enabled()) + pr_info("EEH: PCI Enhanced I/O Error Handling Enabled\n"); + else + pr_info("EEH: No capable adapters found\n"); + } + /* Register OPAL event notifier */ eeh_event_irq = opal_event_request(ilog2(OPAL_EVENT_PCI_ERROR)); if (eeh_event_irq < 0) { @@ -384,8 +392,10 @@ static void *pnv_eeh_probe(struct pci_dn *pdn, void *data) return NULL; /* Skip if we haven't probed yet */ - if (phb->ioda.pe_rmap[config_addr] == IODA_INVALID_PE) + if (phb->ioda.pe_rmap[config_addr] == IODA_INVALID_PE) { + eeh_add_flag(EEH_POSTPONED_PROBE); return NULL; + } /* Initialize eeh device */ edev->class_code = pdn->class_code; diff --git a/arch/powerpc/platforms/powernv/idle.c b/arch/powerpc/platforms/powernv/idle.c index 1c5d0675b43c..35f699ebb662 100644 --- a/arch/powerpc/platforms/powernv/idle.c +++ b/arch/powerpc/platforms/powernv/idle.c @@ -36,6 +36,8 @@ #define P9_STOP_SPR_PSSCR 855 static u32 supported_cpuidle_states; +struct pnv_idle_states_t *pnv_idle_states; +int nr_pnv_idle_states; /* * The default stop state that will be used by ppc_md.power_save @@ -177,11 +179,6 @@ static void pnv_alloc_idle_core_states(void) paca_ptrs[cpu]->core_idle_state_ptr = core_idle_state; paca_ptrs[cpu]->thread_idle_state = PNV_THREAD_RUNNING; paca_ptrs[cpu]->thread_mask = 1 << j; - if (!cpu_has_feature(CPU_FTR_POWER9_DD1)) - continue; - paca_ptrs[cpu]->thread_sibling_pacas = - kmalloc_node(paca_ptr_array_size, - GFP_KERNEL, node); } } @@ -622,48 +619,10 @@ int validate_psscr_val_mask(u64 *psscr_val, u64 *psscr_mask, u32 flags) * @dt_idle_states: Number of idle state entries * Returns 0 on success */ -static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags, - int dt_idle_states) +static int __init pnv_power9_idle_init(void) { - u64 *psscr_val = NULL; - u64 *psscr_mask = NULL; - u32 *residency_ns = NULL; u64 max_residency_ns = 0; - int rc = 0, i; - - psscr_val = kcalloc(dt_idle_states, sizeof(*psscr_val), GFP_KERNEL); - psscr_mask = kcalloc(dt_idle_states, sizeof(*psscr_mask), GFP_KERNEL); - residency_ns = kcalloc(dt_idle_states, sizeof(*residency_ns), - GFP_KERNEL); - - if (!psscr_val || !psscr_mask || !residency_ns) { - rc = -1; - goto out; - } - - if (of_property_read_u64_array(np, - "ibm,cpu-idle-state-psscr", - psscr_val, dt_idle_states)) { - pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); - rc = -1; - goto out; - } - - if (of_property_read_u64_array(np, - "ibm,cpu-idle-state-psscr-mask", - psscr_mask, dt_idle_states)) { - pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n"); - rc = -1; - goto out; - } - - if (of_property_read_u32_array(np, - "ibm,cpu-idle-state-residency-ns", - residency_ns, dt_idle_states)) { - pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-residency-ns in DT\n"); - rc = -1; - goto out; - } + int i; /* * Set pnv_first_deep_stop_state, pnv_deepest_stop_psscr_{val,mask}, @@ -679,33 +638,37 @@ static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags, * the shallowest (OPAL_PM_STOP_INST_FAST) loss-less stop state. */ pnv_first_deep_stop_state = MAX_STOP_STATE; - for (i = 0; i < dt_idle_states; i++) { + for (i = 0; i < nr_pnv_idle_states; i++) { int err; - u64 psscr_rl = psscr_val[i] & PSSCR_RL_MASK; + struct pnv_idle_states_t *state = &pnv_idle_states[i]; + u64 psscr_rl = state->psscr_val & PSSCR_RL_MASK; - if ((flags[i] & OPAL_PM_LOSE_FULL_CONTEXT) && - (pnv_first_deep_stop_state > psscr_rl)) + if ((state->flags & OPAL_PM_LOSE_FULL_CONTEXT) && + pnv_first_deep_stop_state > psscr_rl) pnv_first_deep_stop_state = psscr_rl; - err = validate_psscr_val_mask(&psscr_val[i], &psscr_mask[i], - flags[i]); + err = validate_psscr_val_mask(&state->psscr_val, + &state->psscr_mask, + state->flags); if (err) { - report_invalid_psscr_val(psscr_val[i], err); + report_invalid_psscr_val(state->psscr_val, err); continue; } - if (max_residency_ns < residency_ns[i]) { - max_residency_ns = residency_ns[i]; - pnv_deepest_stop_psscr_val = psscr_val[i]; - pnv_deepest_stop_psscr_mask = psscr_mask[i]; - pnv_deepest_stop_flag = flags[i]; + state->valid = true; + + if (max_residency_ns < state->residency_ns) { + max_residency_ns = state->residency_ns; + pnv_deepest_stop_psscr_val = state->psscr_val; + pnv_deepest_stop_psscr_mask = state->psscr_mask; + pnv_deepest_stop_flag = state->flags; deepest_stop_found = true; } if (!default_stop_found && - (flags[i] & OPAL_PM_STOP_INST_FAST)) { - pnv_default_stop_val = psscr_val[i]; - pnv_default_stop_mask = psscr_mask[i]; + (state->flags & OPAL_PM_STOP_INST_FAST)) { + pnv_default_stop_val = state->psscr_val; + pnv_default_stop_mask = state->psscr_mask; default_stop_found = true; } } @@ -728,11 +691,8 @@ static int __init pnv_power9_idle_init(struct device_node *np, u32 *flags, pr_info("cpuidle-powernv: Requested Level (RL) value of first deep stop = 0x%llx\n", pnv_first_deep_stop_state); -out: - kfree(psscr_val); - kfree(psscr_mask); - kfree(residency_ns); - return rc; + + return 0; } /* @@ -740,50 +700,146 @@ out: */ static void __init pnv_probe_idle_states(void) { - struct device_node *np; - int dt_idle_states; - u32 *flags = NULL; int i; + if (nr_pnv_idle_states < 0) { + pr_warn("cpuidle-powernv: no idle states found in the DT\n"); + return; + } + + if (cpu_has_feature(CPU_FTR_ARCH_300)) { + if (pnv_power9_idle_init()) + return; + } + + for (i = 0; i < nr_pnv_idle_states; i++) + supported_cpuidle_states |= pnv_idle_states[i].flags; +} + +/* + * This function parses device-tree and populates all the information + * into pnv_idle_states structure. It also sets up nr_pnv_idle_states + * which is the number of cpuidle states discovered through device-tree. + */ + +static int pnv_parse_cpuidle_dt(void) +{ + struct device_node *np; + int nr_idle_states, i; + int rc = 0; + u32 *temp_u32; + u64 *temp_u64; + const char **temp_string; + np = of_find_node_by_path("/ibm,opal/power-mgt"); if (!np) { pr_warn("opal: PowerMgmt Node not found\n"); - goto out; + return -ENODEV; } - dt_idle_states = of_property_count_u32_elems(np, - "ibm,cpu-idle-state-flags"); - if (dt_idle_states < 0) { - pr_warn("cpuidle-powernv: no idle states found in the DT\n"); + nr_idle_states = of_property_count_u32_elems(np, + "ibm,cpu-idle-state-flags"); + + pnv_idle_states = kcalloc(nr_idle_states, sizeof(*pnv_idle_states), + GFP_KERNEL); + temp_u32 = kcalloc(nr_idle_states, sizeof(u32), GFP_KERNEL); + temp_u64 = kcalloc(nr_idle_states, sizeof(u64), GFP_KERNEL); + temp_string = kcalloc(nr_idle_states, sizeof(char *), GFP_KERNEL); + + if (!(pnv_idle_states && temp_u32 && temp_u64 && temp_string)) { + pr_err("Could not allocate memory for dt parsing\n"); + rc = -ENOMEM; goto out; } - flags = kcalloc(dt_idle_states, sizeof(*flags), GFP_KERNEL); - - if (of_property_read_u32_array(np, - "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { + /* Read flags */ + if (of_property_read_u32_array(np, "ibm,cpu-idle-state-flags", + temp_u32, nr_idle_states)) { pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-flags in DT\n"); + rc = -EINVAL; + goto out; + } + for (i = 0; i < nr_idle_states; i++) + pnv_idle_states[i].flags = temp_u32[i]; + + /* Read latencies */ + if (of_property_read_u32_array(np, "ibm,cpu-idle-state-latencies-ns", + temp_u32, nr_idle_states)) { + pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n"); + rc = -EINVAL; + goto out; + } + for (i = 0; i < nr_idle_states; i++) + pnv_idle_states[i].latency_ns = temp_u32[i]; + + /* Read residencies */ + if (of_property_read_u32_array(np, "ibm,cpu-idle-state-residency-ns", + temp_u32, nr_idle_states)) { + pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n"); + rc = -EINVAL; goto out; } + for (i = 0; i < nr_idle_states; i++) + pnv_idle_states[i].residency_ns = temp_u32[i]; + /* For power9 */ if (cpu_has_feature(CPU_FTR_ARCH_300)) { - if (pnv_power9_idle_init(np, flags, dt_idle_states)) + /* Read pm_crtl_val */ + if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr", + temp_u64, nr_idle_states)) { + pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); + rc = -EINVAL; + goto out; + } + for (i = 0; i < nr_idle_states; i++) + pnv_idle_states[i].psscr_val = temp_u64[i]; + + /* Read pm_crtl_mask */ + if (of_property_read_u64_array(np, "ibm,cpu-idle-state-psscr-mask", + temp_u64, nr_idle_states)) { + pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr-mask in DT\n"); + rc = -EINVAL; goto out; + } + for (i = 0; i < nr_idle_states; i++) + pnv_idle_states[i].psscr_mask = temp_u64[i]; } - for (i = 0; i < dt_idle_states; i++) - supported_cpuidle_states |= flags[i]; + /* + * power8 specific properties ibm,cpu-idle-state-pmicr-mask and + * ibm,cpu-idle-state-pmicr-val were never used and there is no + * plan to use it in near future. Hence, not parsing these properties + */ + if (of_property_read_string_array(np, "ibm,cpu-idle-state-names", + temp_string, nr_idle_states) < 0) { + pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n"); + rc = -EINVAL; + goto out; + } + for (i = 0; i < nr_idle_states; i++) + strlcpy(pnv_idle_states[i].name, temp_string[i], + PNV_IDLE_NAME_LEN); + nr_pnv_idle_states = nr_idle_states; + rc = 0; out: - kfree(flags); + kfree(temp_u32); + kfree(temp_u64); + kfree(temp_string); + return rc; } + static int __init pnv_init_idle_states(void) { - + int rc = 0; supported_cpuidle_states = 0; + /* In case we error out nr_pnv_idle_states will be zero */ + nr_pnv_idle_states = 0; if (cpuidle_disable != IDLE_NO_OVERRIDE) goto out; - + rc = pnv_parse_cpuidle_dt(); + if (rc) + return rc; pnv_probe_idle_states(); if (!(supported_cpuidle_states & OPAL_PM_SLEEP_ENABLED_ER1)) { @@ -805,29 +861,6 @@ static int __init pnv_init_idle_states(void) pnv_alloc_idle_core_states(); - /* - * For each CPU, record its PACA address in each of it's - * sibling thread's PACA at the slot corresponding to this - * CPU's index in the core. - */ - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { - int cpu; - - pr_info("powernv: idle: Saving PACA pointers of all CPUs in their thread sibling PACA\n"); - for_each_present_cpu(cpu) { - int base_cpu = cpu_first_thread_sibling(cpu); - int idx = cpu_thread_in_core(cpu); - int i; - - for (i = 0; i < threads_per_core; i++) { - int j = base_cpu + i; - - paca_ptrs[j]->thread_sibling_pacas[idx] = - paca_ptrs[cpu]; - } - } - } - if (supported_cpuidle_states & OPAL_PM_NAP_ENABLED) ppc_md.power_save = power7_idle; diff --git a/arch/powerpc/platforms/powernv/memtrace.c b/arch/powerpc/platforms/powernv/memtrace.c index b99283df8584..51dc398ae3f7 100644 --- a/arch/powerpc/platforms/powernv/memtrace.c +++ b/arch/powerpc/platforms/powernv/memtrace.c @@ -47,38 +47,9 @@ static ssize_t memtrace_read(struct file *filp, char __user *ubuf, return simple_read_from_buffer(ubuf, count, ppos, ent->mem, ent->size); } -static bool valid_memtrace_range(struct memtrace_entry *dev, - unsigned long start, unsigned long size) -{ - if ((start >= dev->start) && - ((start + size) <= (dev->start + dev->size))) - return true; - - return false; -} - -static int memtrace_mmap(struct file *filp, struct vm_area_struct *vma) -{ - unsigned long size = vma->vm_end - vma->vm_start; - struct memtrace_entry *dev = filp->private_data; - - if (!valid_memtrace_range(dev, vma->vm_pgoff << PAGE_SHIFT, size)) - return -EINVAL; - - vma->vm_page_prot = pgprot_noncached(vma->vm_page_prot); - - if (remap_pfn_range(vma, vma->vm_start, - vma->vm_pgoff + (dev->start >> PAGE_SHIFT), - size, vma->vm_page_prot)) - return -EAGAIN; - - return 0; -} - static const struct file_operations memtrace_fops = { .llseek = default_llseek, .read = memtrace_read, - .mmap = memtrace_mmap, .open = simple_open, }; @@ -206,8 +177,11 @@ static int memtrace_init_debugfs(void) snprintf(ent->name, 16, "%08x", ent->nid); dir = debugfs_create_dir(ent->name, memtrace_debugfs_dir); - if (!dir) + if (!dir) { + pr_err("Failed to create debugfs directory for node %d\n", + ent->nid); return -1; + } ent->dir = dir; debugfs_create_file("trace", 0400, dir, ent, &memtrace_fops); @@ -218,18 +192,93 @@ static int memtrace_init_debugfs(void) return ret; } +static int online_mem_block(struct memory_block *mem, void *arg) +{ + return device_online(&mem->dev); +} + +/* + * Iterate through the chunks of memory we have removed from the kernel + * and attempt to add them back to the kernel. + */ +static int memtrace_online(void) +{ + int i, ret = 0; + struct memtrace_entry *ent; + + for (i = memtrace_array_nr - 1; i >= 0; i--) { + ent = &memtrace_array[i]; + + /* We have onlined this chunk previously */ + if (ent->nid == -1) + continue; + + /* Remove from io mappings */ + if (ent->mem) { + iounmap(ent->mem); + ent->mem = 0; + } + + if (add_memory(ent->nid, ent->start, ent->size)) { + pr_err("Failed to add trace memory to node %d\n", + ent->nid); + ret += 1; + continue; + } + + /* + * If kernel isn't compiled with the auto online option + * we need to online the memory ourselves. + */ + if (!memhp_auto_online) { + walk_memory_range(PFN_DOWN(ent->start), + PFN_UP(ent->start + ent->size - 1), + NULL, online_mem_block); + } + + /* + * Memory was added successfully so clean up references to it + * so on reentry we can tell that this chunk was added. + */ + debugfs_remove_recursive(ent->dir); + pr_info("Added trace memory back to node %d\n", ent->nid); + ent->size = ent->start = ent->nid = -1; + } + if (ret) + return ret; + + /* If all chunks of memory were added successfully, reset globals */ + kfree(memtrace_array); + memtrace_array = NULL; + memtrace_size = 0; + memtrace_array_nr = 0; + return 0; +} + static int memtrace_enable_set(void *data, u64 val) { - if (memtrace_size) + u64 bytes; + + /* + * Don't attempt to do anything if size isn't aligned to a memory + * block or equal to zero. + */ + bytes = memory_block_size_bytes(); + if (val & (bytes - 1)) { + pr_err("Value must be aligned with 0x%llx\n", bytes); return -EINVAL; + } - if (!val) - return -EINVAL; + /* Re-add/online previously removed/offlined memory */ + if (memtrace_size) { + if (memtrace_online()) + return -EAGAIN; + } - /* Make sure size is aligned to a memory block */ - if (val & (memory_block_size_bytes() - 1)) - return -EINVAL; + if (!val) + return 0; + /* Offline and remove memory */ if (memtrace_init_regions_runtime(val)) return -EINVAL; diff --git a/arch/powerpc/platforms/powernv/npu-dma.c b/arch/powerpc/platforms/powernv/npu-dma.c index 8cdf91f5d3a4..8006c54a91e3 100644 --- a/arch/powerpc/platforms/powernv/npu-dma.c +++ b/arch/powerpc/platforms/powernv/npu-dma.c @@ -17,7 +17,9 @@ #include <linux/pci.h> #include <linux/memblock.h> #include <linux/iommu.h> +#include <linux/debugfs.h> +#include <asm/debugfs.h> #include <asm/tlb.h> #include <asm/powernv.h> #include <asm/reg.h> @@ -44,7 +46,8 @@ static DEFINE_SPINLOCK(npu_context_lock); * entire TLB on the GPU for the given PID rather than each specific address in * the range. */ -#define ATSD_THRESHOLD (2*1024*1024) +static uint64_t atsd_threshold = 2 * 1024 * 1024; +static struct dentry *atsd_threshold_dentry; /* * Other types of TCE cache invalidation are not functional in the @@ -437,8 +440,9 @@ static int get_mmio_atsd_reg(struct npu *npu) int i; for (i = 0; i < npu->mmio_atsd_count; i++) { - if (!test_and_set_bit_lock(i, &npu->mmio_atsd_usage)) - return i; + if (!test_bit(i, &npu->mmio_atsd_usage)) + if (!test_and_set_bit_lock(i, &npu->mmio_atsd_usage)) + return i; } return -ENOSPC; @@ -683,7 +687,7 @@ static void pnv_npu2_mn_invalidate_range(struct mmu_notifier *mn, struct npu_context *npu_context = mn_to_npu_context(mn); unsigned long address; - if (end - start > ATSD_THRESHOLD) { + if (end - start > atsd_threshold) { /* * Just invalidate the entire PID if the address range is too * large. @@ -958,6 +962,11 @@ int pnv_npu2_init(struct pnv_phb *phb) static int npu_index; uint64_t rc = 0; + if (!atsd_threshold_dentry) { + atsd_threshold_dentry = debugfs_create_x64("atsd_threshold", + 0600, powerpc_debugfs_root, &atsd_threshold); + } + phb->npu.nmmu_flush = of_property_read_bool(phb->hose->dn, "ibm,nmmu-flush"); for_each_child_of_node(phb->hose->dn, dn) { diff --git a/arch/powerpc/platforms/powernv/opal-dump.c b/arch/powerpc/platforms/powernv/opal-dump.c index 0dc8fa4e0af2..198143833f00 100644 --- a/arch/powerpc/platforms/powernv/opal-dump.c +++ b/arch/powerpc/platforms/powernv/opal-dump.c @@ -225,13 +225,16 @@ static int64_t dump_read_info(uint32_t *dump_id, uint32_t *dump_size, uint32_t * if (rc == OPAL_PARAMETER) rc = opal_dump_info(&id, &size); + if (rc) { + pr_warn("%s: Failed to get dump info (%d)\n", + __func__, rc); + return rc; + } + *dump_id = be32_to_cpu(id); *dump_size = be32_to_cpu(size); *dump_type = be32_to_cpu(type); - if (rc) - pr_warn("%s: Failed to get dump info (%d)\n", - __func__, rc); return rc; } @@ -368,13 +371,12 @@ static irqreturn_t process_dump(int irq, void *data) { int rc; uint32_t dump_id, dump_size, dump_type; - struct dump_obj *dump; char name[22]; struct kobject *kobj; rc = dump_read_info(&dump_id, &dump_size, &dump_type); if (rc != OPAL_SUCCESS) - return rc; + return IRQ_HANDLED; sprintf(name, "0x%x-0x%x", dump_type, dump_id); @@ -386,12 +388,10 @@ static irqreturn_t process_dump(int irq, void *data) if (kobj) { /* Drop reference added by kset_find_obj() */ kobject_put(kobj); - return 0; + return IRQ_HANDLED; } - dump = create_dump_obj(dump_id, dump_size, dump_type); - if (!dump) - return -1; + create_dump_obj(dump_id, dump_size, dump_type); return IRQ_HANDLED; } diff --git a/arch/powerpc/platforms/powernv/opal-irqchip.c b/arch/powerpc/platforms/powernv/opal-irqchip.c index 605c7e5d52c2..bc97770a67db 100644 --- a/arch/powerpc/platforms/powernv/opal-irqchip.c +++ b/arch/powerpc/platforms/powernv/opal-irqchip.c @@ -22,6 +22,7 @@ #include <linux/kthread.h> #include <linux/delay.h> #include <linux/slab.h> +#include <linux/of_irq.h> #include <asm/machdep.h> #include <asm/opal.h> @@ -38,8 +39,8 @@ struct opal_event_irqchip { }; static struct opal_event_irqchip opal_event_irqchip; static u64 last_outstanding_events; -static unsigned int opal_irq_count; -static unsigned int *opal_irqs; +static int opal_irq_count; +static struct resource *opal_irqs; void opal_handle_events(void) { @@ -165,24 +166,23 @@ void opal_event_shutdown(void) /* First free interrupts, which will also mask them */ for (i = 0; i < opal_irq_count; i++) { - if (!opal_irqs[i]) + if (!opal_irqs || !opal_irqs[i].start) continue; if (in_interrupt() || irqs_disabled()) - disable_irq_nosync(opal_irqs[i]); + disable_irq_nosync(opal_irqs[i].start); else - free_irq(opal_irqs[i], NULL); + free_irq(opal_irqs[i].start, NULL); - opal_irqs[i] = 0; + opal_irqs[i].start = 0; } } int __init opal_event_init(void) { struct device_node *dn, *opal_node; - const char **names; - u32 *irqs; - int i, rc; + bool old_style = false; + int i, rc = 0; opal_node = of_find_node_by_path("/ibm,opal"); if (!opal_node) { @@ -207,67 +207,91 @@ int __init opal_event_init(void) goto out; } - /* Get opal-interrupts property and names if present */ - rc = of_property_count_u32_elems(opal_node, "opal-interrupts"); - if (rc < 0) - goto out; + /* Look for new-style (standard) "interrupts" property */ + opal_irq_count = of_irq_count(opal_node); - opal_irq_count = rc; - pr_debug("Found %d interrupts reserved for OPAL\n", opal_irq_count); + /* Absent ? Look for the old one */ + if (opal_irq_count < 1) { + /* Get opal-interrupts property and names if present */ + rc = of_property_count_u32_elems(opal_node, "opal-interrupts"); + if (rc > 0) + opal_irq_count = rc; + old_style = true; + } - irqs = kcalloc(opal_irq_count, sizeof(*irqs), GFP_KERNEL); - names = kcalloc(opal_irq_count, sizeof(*names), GFP_KERNEL); - opal_irqs = kcalloc(opal_irq_count, sizeof(*opal_irqs), GFP_KERNEL); + /* No interrupts ? Bail out */ + if (!opal_irq_count) + goto out; - if (WARN_ON(!irqs || !names || !opal_irqs)) - goto out_free; + pr_debug("OPAL: Found %d interrupts reserved for OPAL using %s scheme\n", + opal_irq_count, old_style ? "old" : "new"); - rc = of_property_read_u32_array(opal_node, "opal-interrupts", - irqs, opal_irq_count); - if (rc < 0) { - pr_err("Error %d reading opal-interrupts array\n", rc); - goto out_free; + /* Allocate an IRQ resources array */ + opal_irqs = kcalloc(opal_irq_count, sizeof(struct resource), GFP_KERNEL); + if (WARN_ON(!opal_irqs)) { + rc = -ENOMEM; + goto out; } - /* It's not an error for the names to be missing */ - of_property_read_string_array(opal_node, "opal-interrupts-names", - names, opal_irq_count); + /* Build the resources array */ + if (old_style) { + /* Old style "opal-interrupts" property */ + for (i = 0; i < opal_irq_count; i++) { + struct resource *r = &opal_irqs[i]; + const char *name = NULL; + u32 hw_irq; + int virq; + + rc = of_property_read_u32_index(opal_node, "opal-interrupts", + i, &hw_irq); + if (WARN_ON(rc < 0)) { + opal_irq_count = i; + break; + } + of_property_read_string_index(opal_node, "opal-interrupts-names", + i, &name); + virq = irq_create_mapping(NULL, hw_irq); + if (!virq) { + pr_warn("Failed to map OPAL irq 0x%x\n", hw_irq); + continue; + } + r->start = r->end = virq; + r->flags = IORESOURCE_IRQ | IRQ_TYPE_LEVEL_LOW; + r->name = name; + } + } else { + /* new style standard "interrupts" property */ + rc = of_irq_to_resource_table(opal_node, opal_irqs, opal_irq_count); + if (WARN_ON(rc < 0)) { + opal_irq_count = 0; + kfree(opal_irqs); + goto out; + } + if (WARN_ON(rc < opal_irq_count)) + opal_irq_count = rc; + } /* Install interrupt handlers */ for (i = 0; i < opal_irq_count; i++) { - unsigned int virq; - char *name; - - /* Get hardware and virtual IRQ */ - virq = irq_create_mapping(NULL, irqs[i]); - if (!virq) { - pr_warn("Failed to map irq 0x%x\n", irqs[i]); - continue; - } + struct resource *r = &opal_irqs[i]; + const char *name; - if (names[i] && strlen(names[i])) - name = kasprintf(GFP_KERNEL, "opal-%s", names[i]); + /* Prefix name */ + if (r->name && strlen(r->name)) + name = kasprintf(GFP_KERNEL, "opal-%s", r->name); else name = kasprintf(GFP_KERNEL, "opal"); /* Install interrupt handler */ - rc = request_irq(virq, opal_interrupt, IRQF_TRIGGER_LOW, + rc = request_irq(r->start, opal_interrupt, r->flags & IRQD_TRIGGER_MASK, name, NULL); if (rc) { - irq_dispose_mapping(virq); - pr_warn("Error %d requesting irq %d (0x%x)\n", - rc, virq, irqs[i]); + pr_warn("Error %d requesting OPAL irq %d\n", rc, (int)r->start); continue; } - - /* Cache IRQ */ - opal_irqs[i] = virq; } - -out_free: - kfree(irqs); - kfree(names); -out: + rc = 0; + out: of_node_put(opal_node); return rc; } diff --git a/arch/powerpc/platforms/powernv/opal-kmsg.c b/arch/powerpc/platforms/powernv/opal-kmsg.c index 6f1214d4de92..55691950d981 100644 --- a/arch/powerpc/platforms/powernv/opal-kmsg.c +++ b/arch/powerpc/platforms/powernv/opal-kmsg.c @@ -23,12 +23,9 @@ * may not be completely printed. This function does not actually dump the * message, it just ensures that OPAL completely flushes the console buffer. */ -static void force_opal_console_flush(struct kmsg_dumper *dumper, +static void kmsg_dump_opal_console_flush(struct kmsg_dumper *dumper, enum kmsg_dump_reason reason) { - int i; - int64_t ret; - /* * Outside of a panic context the pollers will continue to run, * so we don't need to do any special flushing. @@ -36,32 +33,11 @@ static void force_opal_console_flush(struct kmsg_dumper *dumper, if (reason != KMSG_DUMP_PANIC) return; - if (opal_check_token(OPAL_CONSOLE_FLUSH)) { - ret = opal_console_flush(0); - - if (ret == OPAL_UNSUPPORTED || ret == OPAL_PARAMETER) - return; - - /* Incrementally flush until there's nothing left */ - while (opal_console_flush(0) != OPAL_SUCCESS); - } else { - /* - * If OPAL_CONSOLE_FLUSH is not implemented in the firmware, - * the console can still be flushed by calling the polling - * function enough times to flush the buffer. We don't know - * how much output still needs to be flushed, but we can be - * generous since the kernel is in panic and doesn't need - * to do much else. - */ - printk(KERN_NOTICE "opal: OPAL_CONSOLE_FLUSH missing.\n"); - for (i = 0; i < 1024; i++) { - opal_poll_events(NULL); - } - } + opal_flush_console(0); } static struct kmsg_dumper opal_kmsg_dumper = { - .dump = force_opal_console_flush + .dump = kmsg_dump_opal_console_flush }; void __init opal_kmsg_init(void) diff --git a/arch/powerpc/platforms/powernv/opal-sensor-groups.c b/arch/powerpc/platforms/powernv/opal-sensor-groups.c index 541c9ea04a32..f7d04b6a2d7a 100644 --- a/arch/powerpc/platforms/powernv/opal-sensor-groups.c +++ b/arch/powerpc/platforms/powernv/opal-sensor-groups.c @@ -32,6 +32,34 @@ static struct sensor_group { struct sg_attr *sgattrs; } *sgs; +int sensor_group_enable(u32 handle, bool enable) +{ + struct opal_msg msg; + int token, ret; + + token = opal_async_get_token_interruptible(); + if (token < 0) + return token; + + ret = opal_sensor_group_enable(handle, token, enable); + if (ret == OPAL_ASYNC_COMPLETION) { + ret = opal_async_wait_response(token, &msg); + if (ret) { + pr_devel("Failed to wait for the async response\n"); + ret = -EIO; + goto out; + } + ret = opal_error_code(opal_get_async_rc(msg)); + } else { + ret = opal_error_code(ret); + } + +out: + opal_async_release_token(token); + return ret; +} +EXPORT_SYMBOL_GPL(sensor_group_enable); + static ssize_t sg_store(struct kobject *kobj, struct kobj_attribute *attr, const char *buf, size_t count) { diff --git a/arch/powerpc/platforms/powernv/opal-wrappers.S b/arch/powerpc/platforms/powernv/opal-wrappers.S index a8d9b4089c31..251528231a9e 100644 --- a/arch/powerpc/platforms/powernv/opal-wrappers.S +++ b/arch/powerpc/platforms/powernv/opal-wrappers.S @@ -14,6 +14,8 @@ #include <asm/hvcall.h> #include <asm/asm-offsets.h> #include <asm/opal.h> +#include <asm/asm-compat.h> +#include <asm/feature-fixups.h> .section ".text" @@ -327,3 +329,5 @@ OPAL_CALL(opal_npu_tl_set, OPAL_NPU_TL_SET); OPAL_CALL(opal_pci_get_pbcq_tunnel_bar, OPAL_PCI_GET_PBCQ_TUNNEL_BAR); OPAL_CALL(opal_pci_set_pbcq_tunnel_bar, OPAL_PCI_SET_PBCQ_TUNNEL_BAR); OPAL_CALL(opal_sensor_read_u64, OPAL_SENSOR_READ_U64); +OPAL_CALL(opal_sensor_group_enable, OPAL_SENSOR_GROUP_ENABLE); +OPAL_CALL(opal_nx_coproc_init, OPAL_NX_COPROC_INIT); diff --git a/arch/powerpc/platforms/powernv/opal.c b/arch/powerpc/platforms/powernv/opal.c index 0d539c661748..404c379db168 100644 --- a/arch/powerpc/platforms/powernv/opal.c +++ b/arch/powerpc/platforms/powernv/opal.c @@ -344,70 +344,125 @@ int opal_get_chars(uint32_t vtermno, char *buf, int count) return 0; } -int opal_put_chars(uint32_t vtermno, const char *data, int total_len) +static int __opal_put_chars(uint32_t vtermno, const char *data, int total_len, bool atomic) { - int written = 0; + unsigned long flags = 0 /* shut up gcc */; + int written; __be64 olen; - s64 len, rc; - unsigned long flags; - __be64 evt; + s64 rc; if (!opal.entry) return -ENODEV; - /* We want put_chars to be atomic to avoid mangling of hvsi - * packets. To do that, we first test for room and return - * -EAGAIN if there isn't enough. - * - * Unfortunately, opal_console_write_buffer_space() doesn't - * appear to work on opal v1, so we just assume there is - * enough room and be done with it - */ - spin_lock_irqsave(&opal_write_lock, flags); + if (atomic) + spin_lock_irqsave(&opal_write_lock, flags); rc = opal_console_write_buffer_space(vtermno, &olen); - len = be64_to_cpu(olen); - if (rc || len < total_len) { - spin_unlock_irqrestore(&opal_write_lock, flags); + if (rc || be64_to_cpu(olen) < total_len) { /* Closed -> drop characters */ if (rc) - return total_len; - opal_poll_events(NULL); - return -EAGAIN; + written = total_len; + else + written = -EAGAIN; + goto out; } - /* We still try to handle partial completions, though they - * should no longer happen. - */ - rc = OPAL_BUSY; - while(total_len > 0 && (rc == OPAL_BUSY || - rc == OPAL_BUSY_EVENT || rc == OPAL_SUCCESS)) { - olen = cpu_to_be64(total_len); - rc = opal_console_write(vtermno, &olen, data); - len = be64_to_cpu(olen); - - /* Closed or other error drop */ - if (rc != OPAL_SUCCESS && rc != OPAL_BUSY && - rc != OPAL_BUSY_EVENT) { - written = total_len; - break; + /* Should not get a partial write here because space is available. */ + olen = cpu_to_be64(total_len); + rc = opal_console_write(vtermno, &olen, data); + if (rc == OPAL_BUSY || rc == OPAL_BUSY_EVENT) { + if (rc == OPAL_BUSY_EVENT) { + mdelay(OPAL_BUSY_DELAY_MS); + opal_poll_events(NULL); + } else if (rc == OPAL_BUSY_EVENT) { + mdelay(OPAL_BUSY_DELAY_MS); } - if (rc == OPAL_SUCCESS) { - total_len -= len; - data += len; - written += len; + written = -EAGAIN; + goto out; + } + + /* Closed or other error drop */ + if (rc != OPAL_SUCCESS) { + written = opal_error_code(rc); + goto out; + } + + written = be64_to_cpu(olen); + if (written < total_len) { + if (atomic) { + /* Should not happen */ + pr_warn("atomic console write returned partial " + "len=%d written=%d\n", total_len, written); } - /* This is a bit nasty but we need that for the console to - * flush when there aren't any interrupts. We will clean - * things a bit later to limit that to synchronous path - * such as the kernel console and xmon/udbg + if (!written) + written = -EAGAIN; + } + +out: + if (atomic) + spin_unlock_irqrestore(&opal_write_lock, flags); + + /* In the -EAGAIN case, callers loop, so we have to flush the console + * here in case they have interrupts off (and we don't want to wait + * for async flushing if we can make immediate progress here). If + * necessary the API could be made entirely non-flushing if the + * callers had a ->flush API to use. + */ + if (written == -EAGAIN) + opal_flush_console(vtermno); + + return written; +} + +int opal_put_chars(uint32_t vtermno, const char *data, int total_len) +{ + return __opal_put_chars(vtermno, data, total_len, false); +} + +/* + * opal_put_chars_atomic will not perform partial-writes. Data will be + * atomically written to the terminal or not at all. This is not strictly + * true at the moment because console space can race with OPAL's console + * writes. + */ +int opal_put_chars_atomic(uint32_t vtermno, const char *data, int total_len) +{ + return __opal_put_chars(vtermno, data, total_len, true); +} + +int opal_flush_console(uint32_t vtermno) +{ + s64 rc; + + if (!opal_check_token(OPAL_CONSOLE_FLUSH)) { + __be64 evt; + + WARN_ONCE(1, "opal: OPAL_CONSOLE_FLUSH missing.\n"); + /* + * If OPAL_CONSOLE_FLUSH is not implemented in the firmware, + * the console can still be flushed by calling the polling + * function while it has OPAL_EVENT_CONSOLE_OUTPUT events. */ - do + do { opal_poll_events(&evt); - while(rc == OPAL_SUCCESS && - (be64_to_cpu(evt) & OPAL_EVENT_CONSOLE_OUTPUT)); + } while (be64_to_cpu(evt) & OPAL_EVENT_CONSOLE_OUTPUT); + + return OPAL_SUCCESS; } - spin_unlock_irqrestore(&opal_write_lock, flags); - return written; + + do { + rc = OPAL_BUSY; + while (rc == OPAL_BUSY || rc == OPAL_BUSY_EVENT) { + rc = opal_console_flush(vtermno); + if (rc == OPAL_BUSY_EVENT) { + mdelay(OPAL_BUSY_DELAY_MS); + opal_poll_events(NULL); + } else if (rc == OPAL_BUSY) { + mdelay(OPAL_BUSY_DELAY_MS); + } + } + } while (rc == OPAL_PARTIAL); /* More to flush */ + + return opal_error_code(rc); } static int opal_recover_mce(struct pt_regs *regs, @@ -922,6 +977,7 @@ EXPORT_SYMBOL_GPL(opal_flash_read); EXPORT_SYMBOL_GPL(opal_flash_write); EXPORT_SYMBOL_GPL(opal_flash_erase); EXPORT_SYMBOL_GPL(opal_prd_msg); +EXPORT_SYMBOL_GPL(opal_check_token); /* Convert a region of vmalloc memory to an opal sg list */ struct opal_sg_list *opal_vmalloc_to_sg_list(void *vmalloc_addr, @@ -1034,3 +1090,5 @@ EXPORT_SYMBOL_GPL(opal_write_oppanel_async); EXPORT_SYMBOL_GPL(opal_int_set_mfrr); EXPORT_SYMBOL_GPL(opal_int_eoi); EXPORT_SYMBOL_GPL(opal_error_code); +/* Export the below symbol for NX compression */ +EXPORT_SYMBOL(opal_nx_coproc_init); diff --git a/arch/powerpc/platforms/powernv/pci-cxl.c b/arch/powerpc/platforms/powernv/pci-cxl.c index cee003de63af..1b18111453d7 100644 --- a/arch/powerpc/platforms/powernv/pci-cxl.c +++ b/arch/powerpc/platforms/powernv/pci-cxl.c @@ -8,11 +8,8 @@ */ #include <linux/module.h> -#include <linux/msi.h> -#include <asm/pci-bridge.h> #include <asm/pnv-pci.h> #include <asm/opal.h> -#include <misc/cxl.h> #include "pci.h" @@ -179,199 +176,3 @@ static inline int get_cxl_module(void) #else static inline int get_cxl_module(void) { return 0; } #endif - -/* - * Sets flags and switches the controller ops to enable the cxl kernel api. - * Originally the cxl kernel API operated on a virtual PHB, but certain cards - * such as the Mellanox CX4 use a peer model instead and for these cards the - * cxl kernel api will operate on the real PHB. - */ -int pnv_cxl_enable_phb_kernel_api(struct pci_controller *hose, bool enable) -{ - struct pnv_phb *phb = hose->private_data; - int rc; - - if (!enable) { - /* - * Once cxl mode is enabled on the PHB, there is currently no - * known safe method to disable it again, and trying risks a - * checkstop. If we can find a way to safely disable cxl mode - * in the future we can revisit this, but for now the only sane - * thing to do is to refuse to disable cxl mode: - */ - return -EPERM; - } - - /* - * Hold a reference to the cxl module since several PHB operations now - * depend on it, and it would be insane to allow it to be removed so - * long as we are in this mode (and since we can't safely disable this - * mode once enabled...). - */ - rc = get_cxl_module(); - if (rc) - return rc; - - phb->flags |= PNV_PHB_FLAG_CXL; - hose->controller_ops = pnv_cxl_cx4_ioda_controller_ops; - - return 0; -} -EXPORT_SYMBOL_GPL(pnv_cxl_enable_phb_kernel_api); - -bool pnv_pci_on_cxl_phb(struct pci_dev *dev) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - - return !!(phb->flags & PNV_PHB_FLAG_CXL); -} -EXPORT_SYMBOL_GPL(pnv_pci_on_cxl_phb); - -struct cxl_afu *pnv_cxl_phb_to_afu(struct pci_controller *hose) -{ - struct pnv_phb *phb = hose->private_data; - - return (struct cxl_afu *)phb->cxl_afu; -} -EXPORT_SYMBOL_GPL(pnv_cxl_phb_to_afu); - -void pnv_cxl_phb_set_peer_afu(struct pci_dev *dev, struct cxl_afu *afu) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - - phb->cxl_afu = afu; -} -EXPORT_SYMBOL_GPL(pnv_cxl_phb_set_peer_afu); - -/* - * In the peer cxl model, the XSL/PSL is physical function 0, and will be used - * by other functions on the device for memory access and interrupts. When the - * other functions are enabled we explicitly take a reference on the cxl - * function since they will use it, and allocate a default context associated - * with that function just like the vPHB model of the cxl kernel API. - */ -bool pnv_cxl_enable_device_hook(struct pci_dev *dev) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - struct cxl_afu *afu = phb->cxl_afu; - - if (!pnv_pci_enable_device_hook(dev)) - return false; - - - /* No special handling for the cxl function, which is always PF 0 */ - if (PCI_FUNC(dev->devfn) == 0) - return true; - - if (!afu) { - dev_WARN(&dev->dev, "Attempted to enable function > 0 on CXL PHB without a peer AFU\n"); - return false; - } - - dev_info(&dev->dev, "Enabling function on CXL enabled PHB with peer AFU\n"); - - /* Make sure the peer AFU can't go away while this device is active */ - cxl_afu_get(afu); - - return cxl_pci_associate_default_context(dev, afu); -} - -void pnv_cxl_disable_device(struct pci_dev *dev) -{ - struct pci_controller *hose = pci_bus_to_host(dev->bus); - struct pnv_phb *phb = hose->private_data; - struct cxl_afu *afu = phb->cxl_afu; - - /* No special handling for cxl function: */ - if (PCI_FUNC(dev->devfn) == 0) - return; - - cxl_pci_disable_device(dev); - cxl_afu_put(afu); -} - -/* - * This is a special version of pnv_setup_msi_irqs for cards in cxl mode. This - * function handles setting up the IVTE entries for the XSL to use. - * - * We are currently not filling out the MSIX table, since the only currently - * supported adapter (CX4) uses a custom MSIX table format in cxl mode and it - * is up to their driver to fill that out. In the future we may fill out the - * MSIX table (and change the IVTE entries to be an index to the MSIX table) - * for adapters implementing the Full MSI-X mode described in the CAIA. - */ -int pnv_cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) -{ - struct pci_controller *hose = pci_bus_to_host(pdev->bus); - struct pnv_phb *phb = hose->private_data; - struct msi_desc *entry; - struct cxl_context *ctx = NULL; - unsigned int virq; - int hwirq; - int afu_irq = 0; - int rc; - - if (WARN_ON(!phb) || !phb->msi_bmp.bitmap) - return -ENODEV; - - if (pdev->no_64bit_msi && !phb->msi32_support) - return -ENODEV; - - rc = cxl_cx4_setup_msi_irqs(pdev, nvec, type); - if (rc) - return rc; - - for_each_pci_msi_entry(entry, pdev) { - if (!entry->msi_attrib.is_64 && !phb->msi32_support) { - pr_warn("%s: Supports only 64-bit MSIs\n", - pci_name(pdev)); - return -ENXIO; - } - - hwirq = cxl_next_msi_hwirq(pdev, &ctx, &afu_irq); - if (WARN_ON(hwirq <= 0)) - return (hwirq ? hwirq : -ENOMEM); - - virq = irq_create_mapping(NULL, hwirq); - if (!virq) { - pr_warn("%s: Failed to map cxl mode MSI to linux irq\n", - pci_name(pdev)); - return -ENOMEM; - } - - rc = pnv_cxl_ioda_msi_setup(pdev, hwirq, virq); - if (rc) { - pr_warn("%s: Failed to setup cxl mode MSI\n", pci_name(pdev)); - irq_dispose_mapping(virq); - return rc; - } - - irq_set_msi_desc(virq, entry); - } - - return 0; -} - -void pnv_cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev) -{ - struct pci_controller *hose = pci_bus_to_host(pdev->bus); - struct pnv_phb *phb = hose->private_data; - struct msi_desc *entry; - irq_hw_number_t hwirq; - - if (WARN_ON(!phb)) - return; - - for_each_pci_msi_entry(entry, pdev) { - if (!entry->irq) - continue; - hwirq = virq_to_hw(entry->irq); - irq_set_msi_desc(entry->irq, NULL); - irq_dispose_mapping(entry->irq); - } - - cxl_cx4_teardown_msi_irqs(pdev); -} diff --git a/arch/powerpc/platforms/powernv/pci-ioda-tce.c b/arch/powerpc/platforms/powernv/pci-ioda-tce.c new file mode 100644 index 000000000000..6c5db1acbe8d --- /dev/null +++ b/arch/powerpc/platforms/powernv/pci-ioda-tce.c @@ -0,0 +1,399 @@ +// SPDX-License-Identifier: GPL-2.0+ +/* + * TCE helpers for IODA PCI/PCIe on PowerNV platforms + * + * Copyright 2018 IBM Corp. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#include <linux/kernel.h> +#include <linux/iommu.h> + +#include <asm/iommu.h> +#include <asm/tce.h> +#include "pci.h" + +void pnv_pci_setup_iommu_table(struct iommu_table *tbl, + void *tce_mem, u64 tce_size, + u64 dma_offset, unsigned int page_shift) +{ + tbl->it_blocksize = 16; + tbl->it_base = (unsigned long)tce_mem; + tbl->it_page_shift = page_shift; + tbl->it_offset = dma_offset >> tbl->it_page_shift; + tbl->it_index = 0; + tbl->it_size = tce_size >> 3; + tbl->it_busno = 0; + tbl->it_type = TCE_PCI; +} + +static __be64 *pnv_alloc_tce_level(int nid, unsigned int shift) +{ + struct page *tce_mem = NULL; + __be64 *addr; + + tce_mem = alloc_pages_node(nid, GFP_KERNEL, shift - PAGE_SHIFT); + if (!tce_mem) { + pr_err("Failed to allocate a TCE memory, level shift=%d\n", + shift); + return NULL; + } + addr = page_address(tce_mem); + memset(addr, 0, 1UL << shift); + + return addr; +} + +static __be64 *pnv_tce(struct iommu_table *tbl, bool user, long idx, bool alloc) +{ + __be64 *tmp = user ? tbl->it_userspace : (__be64 *) tbl->it_base; + int level = tbl->it_indirect_levels; + const long shift = ilog2(tbl->it_level_size); + unsigned long mask = (tbl->it_level_size - 1) << (level * shift); + + while (level) { + int n = (idx & mask) >> (level * shift); + unsigned long tce; + + if (tmp[n] == 0) { + __be64 *tmp2; + + if (!alloc) + return NULL; + + tmp2 = pnv_alloc_tce_level(tbl->it_nid, + ilog2(tbl->it_level_size) + 3); + if (!tmp2) + return NULL; + + tmp[n] = cpu_to_be64(__pa(tmp2) | + TCE_PCI_READ | TCE_PCI_WRITE); + } + tce = be64_to_cpu(tmp[n]); + + tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE)); + idx &= ~mask; + mask >>= shift; + --level; + } + + return tmp + idx; +} + +int pnv_tce_build(struct iommu_table *tbl, long index, long npages, + unsigned long uaddr, enum dma_data_direction direction, + unsigned long attrs) +{ + u64 proto_tce = iommu_direction_to_tce_perm(direction); + u64 rpn = __pa(uaddr) >> tbl->it_page_shift; + long i; + + if (proto_tce & TCE_PCI_WRITE) + proto_tce |= TCE_PCI_READ; + + for (i = 0; i < npages; i++) { + unsigned long newtce = proto_tce | + ((rpn + i) << tbl->it_page_shift); + unsigned long idx = index - tbl->it_offset + i; + + *(pnv_tce(tbl, false, idx, true)) = cpu_to_be64(newtce); + } + + return 0; +} + +#ifdef CONFIG_IOMMU_API +int pnv_tce_xchg(struct iommu_table *tbl, long index, + unsigned long *hpa, enum dma_data_direction *direction, + bool alloc) +{ + u64 proto_tce = iommu_direction_to_tce_perm(*direction); + unsigned long newtce = *hpa | proto_tce, oldtce; + unsigned long idx = index - tbl->it_offset; + __be64 *ptce = NULL; + + BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl)); + + if (*direction == DMA_NONE) { + ptce = pnv_tce(tbl, false, idx, false); + if (!ptce) { + *hpa = 0; + return 0; + } + } + + if (!ptce) { + ptce = pnv_tce(tbl, false, idx, alloc); + if (!ptce) + return alloc ? H_HARDWARE : H_TOO_HARD; + } + + if (newtce & TCE_PCI_WRITE) + newtce |= TCE_PCI_READ; + + oldtce = be64_to_cpu(xchg(ptce, cpu_to_be64(newtce))); + *hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE); + *direction = iommu_tce_direction(oldtce); + + return 0; +} + +__be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index, bool alloc) +{ + if (WARN_ON_ONCE(!tbl->it_userspace)) + return NULL; + + return pnv_tce(tbl, true, index - tbl->it_offset, alloc); +} +#endif + +void pnv_tce_free(struct iommu_table *tbl, long index, long npages) +{ + long i; + + for (i = 0; i < npages; i++) { + unsigned long idx = index - tbl->it_offset + i; + __be64 *ptce = pnv_tce(tbl, false, idx, false); + + if (ptce) + *ptce = cpu_to_be64(0); + } +} + +unsigned long pnv_tce_get(struct iommu_table *tbl, long index) +{ + __be64 *ptce = pnv_tce(tbl, false, index - tbl->it_offset, false); + + if (!ptce) + return 0; + + return be64_to_cpu(*ptce); +} + +static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr, + unsigned long size, unsigned int levels) +{ + const unsigned long addr_ul = (unsigned long) addr & + ~(TCE_PCI_READ | TCE_PCI_WRITE); + + if (levels) { + long i; + u64 *tmp = (u64 *) addr_ul; + + for (i = 0; i < size; ++i) { + unsigned long hpa = be64_to_cpu(tmp[i]); + + if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE))) + continue; + + pnv_pci_ioda2_table_do_free_pages(__va(hpa), size, + levels - 1); + } + } + + free_pages(addr_ul, get_order(size << 3)); +} + +void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl) +{ + const unsigned long size = tbl->it_indirect_levels ? + tbl->it_level_size : tbl->it_size; + + if (!tbl->it_size) + return; + + pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size, + tbl->it_indirect_levels); + if (tbl->it_userspace) { + pnv_pci_ioda2_table_do_free_pages(tbl->it_userspace, size, + tbl->it_indirect_levels); + } +} + +static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned int shift, + unsigned int levels, unsigned long limit, + unsigned long *current_offset, unsigned long *total_allocated) +{ + __be64 *addr, *tmp; + unsigned long allocated = 1UL << shift; + unsigned int entries = 1UL << (shift - 3); + long i; + + addr = pnv_alloc_tce_level(nid, shift); + *total_allocated += allocated; + + --levels; + if (!levels) { + *current_offset += allocated; + return addr; + } + + for (i = 0; i < entries; ++i) { + tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift, + levels, limit, current_offset, total_allocated); + if (!tmp) + break; + + addr[i] = cpu_to_be64(__pa(tmp) | + TCE_PCI_READ | TCE_PCI_WRITE); + + if (*current_offset >= limit) + break; + } + + return addr; +} + +long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset, + __u32 page_shift, __u64 window_size, __u32 levels, + bool alloc_userspace_copy, struct iommu_table *tbl) +{ + void *addr, *uas = NULL; + unsigned long offset = 0, level_shift, total_allocated = 0; + unsigned long total_allocated_uas = 0; + const unsigned int window_shift = ilog2(window_size); + unsigned int entries_shift = window_shift - page_shift; + unsigned int table_shift = max_t(unsigned int, entries_shift + 3, + PAGE_SHIFT); + const unsigned long tce_table_size = 1UL << table_shift; + unsigned int tmplevels = levels; + + if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS)) + return -EINVAL; + + if (!is_power_of_2(window_size)) + return -EINVAL; + + if (alloc_userspace_copy && (window_size > (1ULL << 32))) + tmplevels = 1; + + /* Adjust direct table size from window_size and levels */ + entries_shift = (entries_shift + levels - 1) / levels; + level_shift = entries_shift + 3; + level_shift = max_t(unsigned int, level_shift, PAGE_SHIFT); + + if ((level_shift - 3) * levels + page_shift >= 60) + return -EINVAL; + + /* Allocate TCE table */ + addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, + tmplevels, tce_table_size, &offset, &total_allocated); + + /* addr==NULL means that the first level allocation failed */ + if (!addr) + return -ENOMEM; + + /* + * First level was allocated but some lower level failed as + * we did not allocate as much as we wanted, + * release partially allocated table. + */ + if (tmplevels == levels && offset < tce_table_size) + goto free_tces_exit; + + /* Allocate userspace view of the TCE table */ + if (alloc_userspace_copy) { + offset = 0; + uas = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, + levels, tce_table_size, &offset, + &total_allocated_uas); + if (!uas) + goto free_tces_exit; + if (tmplevels == levels && (offset < tce_table_size || + total_allocated_uas != total_allocated)) + goto free_uas_exit; + } + + /* Setup linux iommu table */ + pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset, + page_shift); + tbl->it_level_size = 1ULL << (level_shift - 3); + tbl->it_indirect_levels = levels - 1; + tbl->it_allocated_size = total_allocated; + tbl->it_userspace = uas; + tbl->it_nid = nid; + + pr_debug("Created TCE table: ws=%08llx ts=%lx @%08llx base=%lx uas=%p levels=%d/%d\n", + window_size, tce_table_size, bus_offset, tbl->it_base, + tbl->it_userspace, tmplevels, levels); + + return 0; + +free_uas_exit: + pnv_pci_ioda2_table_do_free_pages(uas, + 1ULL << (level_shift - 3), levels - 1); +free_tces_exit: + pnv_pci_ioda2_table_do_free_pages(addr, + 1ULL << (level_shift - 3), levels - 1); + + return -ENOMEM; +} + +static void pnv_iommu_table_group_link_free(struct rcu_head *head) +{ + struct iommu_table_group_link *tgl = container_of(head, + struct iommu_table_group_link, rcu); + + kfree(tgl); +} + +void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, + struct iommu_table_group *table_group) +{ + long i; + bool found; + struct iommu_table_group_link *tgl; + + if (!tbl || !table_group) + return; + + /* Remove link to a group from table's list of attached groups */ + found = false; + list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { + if (tgl->table_group == table_group) { + list_del_rcu(&tgl->next); + call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free); + found = true; + break; + } + } + if (WARN_ON(!found)) + return; + + /* Clean a pointer to iommu_table in iommu_table_group::tables[] */ + found = false; + for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { + if (table_group->tables[i] == tbl) { + table_group->tables[i] = NULL; + found = true; + break; + } + } + WARN_ON(!found); +} + +long pnv_pci_link_table_and_group(int node, int num, + struct iommu_table *tbl, + struct iommu_table_group *table_group) +{ + struct iommu_table_group_link *tgl = NULL; + + if (WARN_ON(!tbl || !table_group)) + return -EINVAL; + + tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL, + node); + if (!tgl) + return -ENOMEM; + + tgl->table_group = table_group; + list_add_rcu(&tgl->next, &tbl->it_group_list); + + table_group->tables[num] = tbl; + + return 0; +} diff --git a/arch/powerpc/platforms/powernv/pci-ioda.c b/arch/powerpc/platforms/powernv/pci-ioda.c index 70b2e1e0f23c..4e6302bf4073 100644 --- a/arch/powerpc/platforms/powernv/pci-ioda.c +++ b/arch/powerpc/platforms/powernv/pci-ioda.c @@ -52,12 +52,8 @@ #define PNV_IODA1_M64_SEGS 8 /* Segments per M64 BAR */ #define PNV_IODA1_DMA32_SEGSIZE 0x10000000 -#define POWERNV_IOMMU_DEFAULT_LEVELS 1 -#define POWERNV_IOMMU_MAX_LEVELS 5 - static const char * const pnv_phb_names[] = { "IODA1", "IODA2", "NPU_NVLINK", "NPU_OCAPI" }; -static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl); void pe_level_printk(const struct pnv_ioda_pe *pe, const char *level, const char *fmt, ...) @@ -2008,7 +2004,7 @@ static int pnv_ioda1_tce_build(struct iommu_table *tbl, long index, static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index, unsigned long *hpa, enum dma_data_direction *direction) { - long ret = pnv_tce_xchg(tbl, index, hpa, direction); + long ret = pnv_tce_xchg(tbl, index, hpa, direction, true); if (!ret) pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, false); @@ -2019,7 +2015,7 @@ static int pnv_ioda1_tce_xchg(struct iommu_table *tbl, long index, static int pnv_ioda1_tce_xchg_rm(struct iommu_table *tbl, long index, unsigned long *hpa, enum dma_data_direction *direction) { - long ret = pnv_tce_xchg(tbl, index, hpa, direction); + long ret = pnv_tce_xchg(tbl, index, hpa, direction, false); if (!ret) pnv_pci_p7ioc_tce_invalidate(tbl, index, 1, true); @@ -2041,6 +2037,7 @@ static struct iommu_table_ops pnv_ioda1_iommu_ops = { #ifdef CONFIG_IOMMU_API .exchange = pnv_ioda1_tce_xchg, .exchange_rm = pnv_ioda1_tce_xchg_rm, + .useraddrptr = pnv_tce_useraddrptr, #endif .clear = pnv_ioda1_tce_free, .get = pnv_tce_get, @@ -2172,7 +2169,7 @@ static int pnv_ioda2_tce_build(struct iommu_table *tbl, long index, static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index, unsigned long *hpa, enum dma_data_direction *direction) { - long ret = pnv_tce_xchg(tbl, index, hpa, direction); + long ret = pnv_tce_xchg(tbl, index, hpa, direction, true); if (!ret) pnv_pci_ioda2_tce_invalidate(tbl, index, 1, false); @@ -2183,7 +2180,7 @@ static int pnv_ioda2_tce_xchg(struct iommu_table *tbl, long index, static int pnv_ioda2_tce_xchg_rm(struct iommu_table *tbl, long index, unsigned long *hpa, enum dma_data_direction *direction) { - long ret = pnv_tce_xchg(tbl, index, hpa, direction); + long ret = pnv_tce_xchg(tbl, index, hpa, direction, false); if (!ret) pnv_pci_ioda2_tce_invalidate(tbl, index, 1, true); @@ -2200,20 +2197,16 @@ static void pnv_ioda2_tce_free(struct iommu_table *tbl, long index, pnv_pci_ioda2_tce_invalidate(tbl, index, npages, false); } -static void pnv_ioda2_table_free(struct iommu_table *tbl) -{ - pnv_pci_ioda2_table_free_pages(tbl); -} - static struct iommu_table_ops pnv_ioda2_iommu_ops = { .set = pnv_ioda2_tce_build, #ifdef CONFIG_IOMMU_API .exchange = pnv_ioda2_tce_xchg, .exchange_rm = pnv_ioda2_tce_xchg_rm, + .useraddrptr = pnv_tce_useraddrptr, #endif .clear = pnv_ioda2_tce_free, .get = pnv_tce_get, - .free = pnv_ioda2_table_free, + .free = pnv_pci_ioda2_table_free_pages, }; static int pnv_pci_ioda_dev_dma_weight(struct pci_dev *dev, void *data) @@ -2463,13 +2456,9 @@ void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable) pe->tce_bypass_enabled = enable; } -static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset, - __u32 page_shift, __u64 window_size, __u32 levels, - struct iommu_table *tbl); - static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group, int num, __u32 page_shift, __u64 window_size, __u32 levels, - struct iommu_table **ptbl) + bool alloc_userspace_copy, struct iommu_table **ptbl) { struct pnv_ioda_pe *pe = container_of(table_group, struct pnv_ioda_pe, table_group); @@ -2486,7 +2475,7 @@ static long pnv_pci_ioda2_create_table(struct iommu_table_group *table_group, ret = pnv_pci_ioda2_table_alloc_pages(nid, bus_offset, page_shift, window_size, - levels, tbl); + levels, alloc_userspace_copy, tbl); if (ret) { iommu_tce_table_put(tbl); return ret; @@ -2519,7 +2508,7 @@ static long pnv_pci_ioda2_setup_default_config(struct pnv_ioda_pe *pe) rc = pnv_pci_ioda2_create_table(&pe->table_group, 0, IOMMU_PAGE_SHIFT_4K, window_size, - POWERNV_IOMMU_DEFAULT_LEVELS, &tbl); + POWERNV_IOMMU_DEFAULT_LEVELS, false, &tbl); if (rc) { pe_err(pe, "Failed to create 32-bit TCE table, err %ld", rc); @@ -2606,7 +2595,16 @@ static unsigned long pnv_pci_ioda2_get_table_size(__u32 page_shift, tce_table_size, direct_table_size); } - return bytes; + return bytes + bytes; /* one for HW table, one for userspace copy */ +} + +static long pnv_pci_ioda2_create_table_userspace( + struct iommu_table_group *table_group, + int num, __u32 page_shift, __u64 window_size, __u32 levels, + struct iommu_table **ptbl) +{ + return pnv_pci_ioda2_create_table(table_group, + num, page_shift, window_size, levels, true, ptbl); } static void pnv_ioda2_take_ownership(struct iommu_table_group *table_group) @@ -2635,7 +2633,7 @@ static void pnv_ioda2_release_ownership(struct iommu_table_group *table_group) static struct iommu_table_group_ops pnv_pci_ioda2_ops = { .get_table_size = pnv_pci_ioda2_get_table_size, - .create_table = pnv_pci_ioda2_create_table, + .create_table = pnv_pci_ioda2_create_table_userspace, .set_window = pnv_pci_ioda2_set_window, .unset_window = pnv_pci_ioda2_unset_window, .take_ownership = pnv_ioda2_take_ownership, @@ -2740,7 +2738,7 @@ static void pnv_ioda2_npu_take_ownership(struct iommu_table_group *table_group) static struct iommu_table_group_ops pnv_pci_ioda2_npu_ops = { .get_table_size = pnv_pci_ioda2_get_table_size, - .create_table = pnv_pci_ioda2_create_table, + .create_table = pnv_pci_ioda2_create_table_userspace, .set_window = pnv_pci_ioda2_npu_set_window, .unset_window = pnv_pci_ioda2_npu_unset_window, .take_ownership = pnv_ioda2_npu_take_ownership, @@ -2774,144 +2772,6 @@ static void pnv_pci_ioda_setup_iommu_api(void) static void pnv_pci_ioda_setup_iommu_api(void) { }; #endif -static __be64 *pnv_pci_ioda2_table_do_alloc_pages(int nid, unsigned shift, - unsigned levels, unsigned long limit, - unsigned long *current_offset, unsigned long *total_allocated) -{ - struct page *tce_mem = NULL; - __be64 *addr, *tmp; - unsigned order = max_t(unsigned, shift, PAGE_SHIFT) - PAGE_SHIFT; - unsigned long allocated = 1UL << (order + PAGE_SHIFT); - unsigned entries = 1UL << (shift - 3); - long i; - - tce_mem = alloc_pages_node(nid, GFP_KERNEL, order); - if (!tce_mem) { - pr_err("Failed to allocate a TCE memory, order=%d\n", order); - return NULL; - } - addr = page_address(tce_mem); - memset(addr, 0, allocated); - *total_allocated += allocated; - - --levels; - if (!levels) { - *current_offset += allocated; - return addr; - } - - for (i = 0; i < entries; ++i) { - tmp = pnv_pci_ioda2_table_do_alloc_pages(nid, shift, - levels, limit, current_offset, total_allocated); - if (!tmp) - break; - - addr[i] = cpu_to_be64(__pa(tmp) | - TCE_PCI_READ | TCE_PCI_WRITE); - - if (*current_offset >= limit) - break; - } - - return addr; -} - -static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr, - unsigned long size, unsigned level); - -static long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset, - __u32 page_shift, __u64 window_size, __u32 levels, - struct iommu_table *tbl) -{ - void *addr; - unsigned long offset = 0, level_shift, total_allocated = 0; - const unsigned window_shift = ilog2(window_size); - unsigned entries_shift = window_shift - page_shift; - unsigned table_shift = max_t(unsigned, entries_shift + 3, PAGE_SHIFT); - const unsigned long tce_table_size = 1UL << table_shift; - - if (!levels || (levels > POWERNV_IOMMU_MAX_LEVELS)) - return -EINVAL; - - if (!is_power_of_2(window_size)) - return -EINVAL; - - /* Adjust direct table size from window_size and levels */ - entries_shift = (entries_shift + levels - 1) / levels; - level_shift = entries_shift + 3; - level_shift = max_t(unsigned, level_shift, PAGE_SHIFT); - - if ((level_shift - 3) * levels + page_shift >= 60) - return -EINVAL; - - /* Allocate TCE table */ - addr = pnv_pci_ioda2_table_do_alloc_pages(nid, level_shift, - levels, tce_table_size, &offset, &total_allocated); - - /* addr==NULL means that the first level allocation failed */ - if (!addr) - return -ENOMEM; - - /* - * First level was allocated but some lower level failed as - * we did not allocate as much as we wanted, - * release partially allocated table. - */ - if (offset < tce_table_size) { - pnv_pci_ioda2_table_do_free_pages(addr, - 1ULL << (level_shift - 3), levels - 1); - return -ENOMEM; - } - - /* Setup linux iommu table */ - pnv_pci_setup_iommu_table(tbl, addr, tce_table_size, bus_offset, - page_shift); - tbl->it_level_size = 1ULL << (level_shift - 3); - tbl->it_indirect_levels = levels - 1; - tbl->it_allocated_size = total_allocated; - - pr_devel("Created TCE table: ws=%08llx ts=%lx @%08llx\n", - window_size, tce_table_size, bus_offset); - - return 0; -} - -static void pnv_pci_ioda2_table_do_free_pages(__be64 *addr, - unsigned long size, unsigned level) -{ - const unsigned long addr_ul = (unsigned long) addr & - ~(TCE_PCI_READ | TCE_PCI_WRITE); - - if (level) { - long i; - u64 *tmp = (u64 *) addr_ul; - - for (i = 0; i < size; ++i) { - unsigned long hpa = be64_to_cpu(tmp[i]); - - if (!(hpa & (TCE_PCI_READ | TCE_PCI_WRITE))) - continue; - - pnv_pci_ioda2_table_do_free_pages(__va(hpa), size, - level - 1); - } - } - - free_pages(addr_ul, get_order(size << 3)); -} - -static void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl) -{ - const unsigned long size = tbl->it_indirect_levels ? - tbl->it_level_size : tbl->it_size; - - if (!tbl->it_size) - return; - - pnv_pci_ioda2_table_do_free_pages((__be64 *)tbl->it_base, size, - tbl->it_indirect_levels); -} - static unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb) { struct pci_controller *hose = phb->hose; @@ -2926,7 +2786,7 @@ static unsigned long pnv_ioda_parse_tce_sizes(struct pnv_phb *phb) /* Add 16M for POWER8 by default */ if (cpu_has_feature(CPU_FTR_ARCH_207S) && !cpu_has_feature(CPU_FTR_ARCH_300)) - mask |= SZ_16M; + mask |= SZ_16M | SZ_256M; return mask; } @@ -3576,7 +3436,7 @@ static resource_size_t pnv_pci_iov_resource_alignment(struct pci_dev *pdev, /* Prevent enabling devices for which we couldn't properly * assign a PE */ -bool pnv_pci_enable_device_hook(struct pci_dev *dev) +static bool pnv_pci_enable_device_hook(struct pci_dev *dev) { struct pci_controller *hose = pci_bus_to_host(dev->bus); struct pnv_phb *phb = hose->private_data; @@ -3844,26 +3704,6 @@ static const struct pci_controller_ops pnv_npu_ocapi_ioda_controller_ops = { .shutdown = pnv_pci_ioda_shutdown, }; -#ifdef CONFIG_CXL_BASE -const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops = { - .dma_dev_setup = pnv_pci_dma_dev_setup, - .dma_bus_setup = pnv_pci_dma_bus_setup, -#ifdef CONFIG_PCI_MSI - .setup_msi_irqs = pnv_cxl_cx4_setup_msi_irqs, - .teardown_msi_irqs = pnv_cxl_cx4_teardown_msi_irqs, -#endif - .enable_device_hook = pnv_cxl_enable_device_hook, - .disable_device = pnv_cxl_disable_device, - .release_device = pnv_pci_release_device, - .window_alignment = pnv_pci_window_alignment, - .setup_bridge = pnv_pci_setup_bridge, - .reset_secondary_bus = pnv_pci_reset_secondary_bus, - .dma_set_mask = pnv_pci_ioda_dma_set_mask, - .dma_get_required_mask = pnv_pci_ioda_dma_get_required_mask, - .shutdown = pnv_pci_ioda_shutdown, -}; -#endif - static void __init pnv_pci_init_ioda_phb(struct device_node *np, u64 hub_id, int ioda_type) { diff --git a/arch/powerpc/platforms/powernv/pci.c b/arch/powerpc/platforms/powernv/pci.c index b265ecc0836a..13aef2323bbc 100644 --- a/arch/powerpc/platforms/powernv/pci.c +++ b/arch/powerpc/platforms/powernv/pci.c @@ -802,85 +802,6 @@ struct pci_ops pnv_pci_ops = { .write = pnv_pci_write_config, }; -static __be64 *pnv_tce(struct iommu_table *tbl, long idx) -{ - __be64 *tmp = ((__be64 *)tbl->it_base); - int level = tbl->it_indirect_levels; - const long shift = ilog2(tbl->it_level_size); - unsigned long mask = (tbl->it_level_size - 1) << (level * shift); - - while (level) { - int n = (idx & mask) >> (level * shift); - unsigned long tce = be64_to_cpu(tmp[n]); - - tmp = __va(tce & ~(TCE_PCI_READ | TCE_PCI_WRITE)); - idx &= ~mask; - mask >>= shift; - --level; - } - - return tmp + idx; -} - -int pnv_tce_build(struct iommu_table *tbl, long index, long npages, - unsigned long uaddr, enum dma_data_direction direction, - unsigned long attrs) -{ - u64 proto_tce = iommu_direction_to_tce_perm(direction); - u64 rpn = __pa(uaddr) >> tbl->it_page_shift; - long i; - - if (proto_tce & TCE_PCI_WRITE) - proto_tce |= TCE_PCI_READ; - - for (i = 0; i < npages; i++) { - unsigned long newtce = proto_tce | - ((rpn + i) << tbl->it_page_shift); - unsigned long idx = index - tbl->it_offset + i; - - *(pnv_tce(tbl, idx)) = cpu_to_be64(newtce); - } - - return 0; -} - -#ifdef CONFIG_IOMMU_API -int pnv_tce_xchg(struct iommu_table *tbl, long index, - unsigned long *hpa, enum dma_data_direction *direction) -{ - u64 proto_tce = iommu_direction_to_tce_perm(*direction); - unsigned long newtce = *hpa | proto_tce, oldtce; - unsigned long idx = index - tbl->it_offset; - - BUG_ON(*hpa & ~IOMMU_PAGE_MASK(tbl)); - - if (newtce & TCE_PCI_WRITE) - newtce |= TCE_PCI_READ; - - oldtce = be64_to_cpu(xchg(pnv_tce(tbl, idx), cpu_to_be64(newtce))); - *hpa = oldtce & ~(TCE_PCI_READ | TCE_PCI_WRITE); - *direction = iommu_tce_direction(oldtce); - - return 0; -} -#endif - -void pnv_tce_free(struct iommu_table *tbl, long index, long npages) -{ - long i; - - for (i = 0; i < npages; i++) { - unsigned long idx = index - tbl->it_offset + i; - - *(pnv_tce(tbl, idx)) = cpu_to_be64(0); - } -} - -unsigned long pnv_tce_get(struct iommu_table *tbl, long index) -{ - return be64_to_cpu(*(pnv_tce(tbl, index - tbl->it_offset))); -} - struct iommu_table *pnv_pci_table_alloc(int nid) { struct iommu_table *tbl; @@ -895,85 +816,6 @@ struct iommu_table *pnv_pci_table_alloc(int nid) return tbl; } -long pnv_pci_link_table_and_group(int node, int num, - struct iommu_table *tbl, - struct iommu_table_group *table_group) -{ - struct iommu_table_group_link *tgl = NULL; - - if (WARN_ON(!tbl || !table_group)) - return -EINVAL; - - tgl = kzalloc_node(sizeof(struct iommu_table_group_link), GFP_KERNEL, - node); - if (!tgl) - return -ENOMEM; - - tgl->table_group = table_group; - list_add_rcu(&tgl->next, &tbl->it_group_list); - - table_group->tables[num] = tbl; - - return 0; -} - -static void pnv_iommu_table_group_link_free(struct rcu_head *head) -{ - struct iommu_table_group_link *tgl = container_of(head, - struct iommu_table_group_link, rcu); - - kfree(tgl); -} - -void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, - struct iommu_table_group *table_group) -{ - long i; - bool found; - struct iommu_table_group_link *tgl; - - if (!tbl || !table_group) - return; - - /* Remove link to a group from table's list of attached groups */ - found = false; - list_for_each_entry_rcu(tgl, &tbl->it_group_list, next) { - if (tgl->table_group == table_group) { - list_del_rcu(&tgl->next); - call_rcu(&tgl->rcu, pnv_iommu_table_group_link_free); - found = true; - break; - } - } - if (WARN_ON(!found)) - return; - - /* Clean a pointer to iommu_table in iommu_table_group::tables[] */ - found = false; - for (i = 0; i < IOMMU_TABLE_GROUP_MAX_TABLES; ++i) { - if (table_group->tables[i] == tbl) { - table_group->tables[i] = NULL; - found = true; - break; - } - } - WARN_ON(!found); -} - -void pnv_pci_setup_iommu_table(struct iommu_table *tbl, - void *tce_mem, u64 tce_size, - u64 dma_offset, unsigned page_shift) -{ - tbl->it_blocksize = 16; - tbl->it_base = (unsigned long)tce_mem; - tbl->it_page_shift = page_shift; - tbl->it_offset = dma_offset >> tbl->it_page_shift; - tbl->it_index = 0; - tbl->it_size = tce_size >> 3; - tbl->it_busno = 0; - tbl->it_type = TCE_PCI; -} - void pnv_pci_dma_dev_setup(struct pci_dev *pdev) { struct pci_controller *hose = pci_bus_to_host(pdev->bus); diff --git a/arch/powerpc/platforms/powernv/pci.h b/arch/powerpc/platforms/powernv/pci.h index eada4b6068cb..8b37b28e3831 100644 --- a/arch/powerpc/platforms/powernv/pci.h +++ b/arch/powerpc/platforms/powernv/pci.h @@ -88,7 +88,6 @@ struct pnv_ioda_pe { }; #define PNV_PHB_FLAG_EEH (1 << 0) -#define PNV_PHB_FLAG_CXL (1 << 1) /* Real PHB supporting the cxl kernel API */ struct pnv_phb { struct pci_controller *hose; @@ -194,20 +193,10 @@ struct pnv_phb { bool nmmu_flush; } npu; -#ifdef CONFIG_CXL_BASE - struct cxl_afu *cxl_afu; -#endif int p2p_target_count; }; extern struct pci_ops pnv_pci_ops; -extern int pnv_tce_build(struct iommu_table *tbl, long index, long npages, - unsigned long uaddr, enum dma_data_direction direction, - unsigned long attrs); -extern void pnv_tce_free(struct iommu_table *tbl, long index, long npages); -extern int pnv_tce_xchg(struct iommu_table *tbl, long index, - unsigned long *hpa, enum dma_data_direction *direction); -extern unsigned long pnv_tce_get(struct iommu_table *tbl, long index); void pnv_pci_dump_phb_diag_data(struct pci_controller *hose, unsigned char *log_buff); @@ -217,14 +206,6 @@ int pnv_pci_cfg_write(struct pci_dn *pdn, int where, int size, u32 val); extern struct iommu_table *pnv_pci_table_alloc(int nid); -extern long pnv_pci_link_table_and_group(int node, int num, - struct iommu_table *tbl, - struct iommu_table_group *table_group); -extern void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, - struct iommu_table_group *table_group); -extern void pnv_pci_setup_iommu_table(struct iommu_table *tbl, - void *tce_mem, u64 tce_size, - u64 dma_offset, unsigned page_shift); extern void pnv_pci_init_ioda_hub(struct device_node *np); extern void pnv_pci_init_ioda2_phb(struct device_node *np); extern void pnv_pci_init_npu_phb(struct device_node *np); @@ -238,7 +219,6 @@ extern int pnv_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type); extern void pnv_teardown_msi_irqs(struct pci_dev *pdev); extern struct pnv_ioda_pe *pnv_ioda_get_pe(struct pci_dev *dev); extern void pnv_set_msi_irq_chip(struct pnv_phb *phb, unsigned int virq); -extern bool pnv_pci_enable_device_hook(struct pci_dev *dev); extern void pnv_pci_ioda2_set_bypass(struct pnv_ioda_pe *pe, bool enable); extern int pnv_eeh_post_init(void); @@ -262,14 +242,33 @@ extern void pnv_npu_take_ownership(struct pnv_ioda_pe *npe); extern void pnv_npu_release_ownership(struct pnv_ioda_pe *npe); extern int pnv_npu2_init(struct pnv_phb *phb); -/* cxl functions */ -extern bool pnv_cxl_enable_device_hook(struct pci_dev *dev); -extern void pnv_cxl_disable_device(struct pci_dev *dev); -extern int pnv_cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type); -extern void pnv_cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev); +/* pci-ioda-tce.c */ +#define POWERNV_IOMMU_DEFAULT_LEVELS 1 +#define POWERNV_IOMMU_MAX_LEVELS 5 +extern int pnv_tce_build(struct iommu_table *tbl, long index, long npages, + unsigned long uaddr, enum dma_data_direction direction, + unsigned long attrs); +extern void pnv_tce_free(struct iommu_table *tbl, long index, long npages); +extern int pnv_tce_xchg(struct iommu_table *tbl, long index, + unsigned long *hpa, enum dma_data_direction *direction, + bool alloc); +extern __be64 *pnv_tce_useraddrptr(struct iommu_table *tbl, long index, + bool alloc); +extern unsigned long pnv_tce_get(struct iommu_table *tbl, long index); + +extern long pnv_pci_ioda2_table_alloc_pages(int nid, __u64 bus_offset, + __u32 page_shift, __u64 window_size, __u32 levels, + bool alloc_userspace_copy, struct iommu_table *tbl); +extern void pnv_pci_ioda2_table_free_pages(struct iommu_table *tbl); -/* phb ops (cxl switches these when enabling the kernel api on the phb) */ -extern const struct pci_controller_ops pnv_cxl_cx4_ioda_controller_ops; +extern long pnv_pci_link_table_and_group(int node, int num, + struct iommu_table *tbl, + struct iommu_table_group *table_group); +extern void pnv_pci_unlink_table_and_group(struct iommu_table *tbl, + struct iommu_table_group *table_group); +extern void pnv_pci_setup_iommu_table(struct iommu_table *tbl, + void *tce_mem, u64 tce_size, + u64 dma_offset, unsigned int page_shift); #endif /* __POWERNV_PCI_H */ diff --git a/arch/powerpc/platforms/powernv/setup.c b/arch/powerpc/platforms/powernv/setup.c index f96df0a25d05..adddde023622 100644 --- a/arch/powerpc/platforms/powernv/setup.c +++ b/arch/powerpc/platforms/powernv/setup.c @@ -78,6 +78,12 @@ static void init_fw_feat_flags(struct device_node *np) if (fw_feature_is("enabled", "fw-count-cache-disabled", np)) security_ftr_set(SEC_FTR_COUNT_CACHE_DISABLED); + if (fw_feature_is("enabled", "fw-count-cache-flush-bcctr2,0,0", np)) + security_ftr_set(SEC_FTR_BCCTR_FLUSH_ASSIST); + + if (fw_feature_is("enabled", "needs-count-cache-flush-on-context-switch", np)) + security_ftr_set(SEC_FTR_FLUSH_COUNT_CACHE); + /* * The features below are enabled by default, so we instead look to see * if firmware has *disabled* them, and clear them if so. @@ -124,7 +130,7 @@ static void pnv_setup_rfi_flush(void) security_ftr_enabled(SEC_FTR_L1D_FLUSH_HV)); setup_rfi_flush(type, enable); - setup_barrier_nospec(); + setup_count_cache_flush(); } static void __init pnv_setup_arch(void) @@ -314,7 +320,7 @@ static void pnv_kexec_cpu_down(int crash_shutdown, int secondary) u64 reinit_flags; if (xive_enabled()) - xive_kexec_teardown_cpu(secondary); + xive_teardown_cpu(); else xics_kexec_teardown_cpu(secondary); diff --git a/arch/powerpc/platforms/powernv/smp.c b/arch/powerpc/platforms/powernv/smp.c index b80909957792..0d354e19ef92 100644 --- a/arch/powerpc/platforms/powernv/smp.c +++ b/arch/powerpc/platforms/powernv/smp.c @@ -283,23 +283,6 @@ static void pnv_cause_ipi(int cpu) ic_cause_ipi(cpu); } -static void pnv_p9_dd1_cause_ipi(int cpu) -{ - int this_cpu = get_cpu(); - - /* - * POWER9 DD1 has a global addressed msgsnd, but for now we restrict - * IPIs to same core, because it requires additional synchronization - * for inter-core doorbells which we do not implement. - */ - if (cpumask_test_cpu(cpu, cpu_sibling_mask(this_cpu))) - doorbell_global_ipi(cpu); - else - ic_cause_ipi(cpu); - - put_cpu(); -} - static void __init pnv_smp_probe(void) { if (xive_enabled()) @@ -311,14 +294,10 @@ static void __init pnv_smp_probe(void) ic_cause_ipi = smp_ops->cause_ipi; WARN_ON(!ic_cause_ipi); - if (cpu_has_feature(CPU_FTR_ARCH_300)) { - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) - smp_ops->cause_ipi = pnv_p9_dd1_cause_ipi; - else - smp_ops->cause_ipi = doorbell_global_ipi; - } else { + if (cpu_has_feature(CPU_FTR_ARCH_300)) + smp_ops->cause_ipi = doorbell_global_ipi; + else smp_ops->cause_ipi = pnv_cause_ipi; - } } } diff --git a/arch/powerpc/platforms/powernv/vas.h b/arch/powerpc/platforms/powernv/vas.h index ae0100fd35bb..f5493dbdd7ff 100644 --- a/arch/powerpc/platforms/powernv/vas.h +++ b/arch/powerpc/platforms/powernv/vas.h @@ -15,6 +15,7 @@ #include <linux/io.h> #include <linux/dcache.h> #include <linux/mutex.h> +#include <linux/stringify.h> /* * Overview of Virtual Accelerator Switchboard (VAS). diff --git a/arch/powerpc/platforms/pseries/Makefile b/arch/powerpc/platforms/pseries/Makefile index 13eede60c24d..7e89d5c47068 100644 --- a/arch/powerpc/platforms/pseries/Makefile +++ b/arch/powerpc/platforms/pseries/Makefile @@ -25,6 +25,6 @@ obj-$(CONFIG_LPARCFG) += lparcfg.o obj-$(CONFIG_IBMVIO) += vio.o obj-$(CONFIG_IBMEBUS) += ibmebus.o -ifeq ($(CONFIG_PPC_PSERIES),y) +ifdef CONFIG_PPC_PSERIES obj-$(CONFIG_SUSPEND) += suspend.o endif diff --git a/arch/powerpc/platforms/pseries/hvCall.S b/arch/powerpc/platforms/pseries/hvCall.S index c511a1743a44..d91412c591ef 100644 --- a/arch/powerpc/platforms/pseries/hvCall.S +++ b/arch/powerpc/platforms/pseries/hvCall.S @@ -13,6 +13,7 @@ #include <asm/ppc_asm.h> #include <asm/asm-offsets.h> #include <asm/ptrace.h> +#include <asm/feature-fixups.h> .section ".text" diff --git a/arch/powerpc/platforms/pseries/kexec.c b/arch/powerpc/platforms/pseries/kexec.c index 46fbaef69a59..23f54223ed56 100644 --- a/arch/powerpc/platforms/pseries/kexec.c +++ b/arch/powerpc/platforms/pseries/kexec.c @@ -58,7 +58,7 @@ void pseries_kexec_cpu_down(int crash_shutdown, int secondary) } if (xive_enabled()) { - xive_kexec_teardown_cpu(secondary); + xive_teardown_cpu(); if (!secondary) xive_shutdown(); diff --git a/arch/powerpc/platforms/pseries/lpar.c b/arch/powerpc/platforms/pseries/lpar.c index 5a392e40f3d2..d3992ced0782 100644 --- a/arch/powerpc/platforms/pseries/lpar.c +++ b/arch/powerpc/platforms/pseries/lpar.c @@ -21,6 +21,7 @@ /* Enables debugging of low-level hash table routines - careful! */ #undef DEBUG +#define pr_fmt(fmt) "lpar: " fmt #include <linux/kernel.h> #include <linux/dma-mapping.h> @@ -36,7 +37,6 @@ #include <asm/machdep.h> #include <asm/mmu_context.h> #include <asm/iommu.h> -#include <asm/tlbflush.h> #include <asm/tlb.h> #include <asm/prom.h> #include <asm/cputable.h> @@ -165,8 +165,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group, lpar_rc = plpar_pte_enter(flags, hpte_group, hpte_v, hpte_r, &slot); if (unlikely(lpar_rc == H_PTEG_FULL)) { - if (!(vflags & HPTE_V_BOLTED)) - pr_devel(" full\n"); + pr_devel("Hash table group is full\n"); return -1; } @@ -176,8 +175,7 @@ static long pSeries_lpar_hpte_insert(unsigned long hpte_group, * or we will loop forever, so return -2 in this case. */ if (unlikely(lpar_rc != H_SUCCESS)) { - if (!(vflags & HPTE_V_BOLTED)) - pr_devel(" lpar err %ld\n", lpar_rc); + pr_err("Failed hash pte insert with error %ld\n", lpar_rc); return -2; } if (!(vflags & HPTE_V_BOLTED)) @@ -240,8 +238,11 @@ static void manual_hpte_clear_all(void) */ for (i = 0; i < hpte_count; i += 4) { lpar_rc = plpar_pte_read_4_raw(0, i, (void *)ptes); - if (lpar_rc != H_SUCCESS) + if (lpar_rc != H_SUCCESS) { + pr_info("Failed to read hash page table at %ld err %ld\n", + i, lpar_rc); continue; + } for (j = 0; j < 4; j++){ if ((ptes[j].pteh & HPTE_V_VRMA_MASK) == HPTE_V_VRMA_MASK) @@ -340,8 +341,11 @@ static long __pSeries_lpar_hpte_find(unsigned long want_v, unsigned long hpte_gr for (i = 0; i < HPTES_PER_GROUP; i += 4, hpte_group += 4) { lpar_rc = plpar_pte_read_4(0, hpte_group, (void *)ptes); - if (lpar_rc != H_SUCCESS) + if (lpar_rc != H_SUCCESS) { + pr_info("Failed to read hash page table at %ld err %ld\n", + hpte_group, lpar_rc); continue; + } for (j = 0; j < 4; j++) { if (HPTE_V_COMPARE(ptes[j].pteh, want_v) && @@ -612,8 +616,8 @@ static int __init disable_bulk_remove(char *str) { if (strcmp(str, "off") == 0 && firmware_has_feature(FW_FEATURE_BULK_REMOVE)) { - printk(KERN_INFO "Disabling BULK_REMOVE firmware feature"); - powerpc_firmware_features &= ~FW_FEATURE_BULK_REMOVE; + pr_info("Disabling BULK_REMOVE firmware feature"); + powerpc_firmware_features &= ~FW_FEATURE_BULK_REMOVE; } return 1; } @@ -659,8 +663,7 @@ static int pseries_lpar_resize_hpt(unsigned long shift) if (!firmware_has_feature(FW_FEATURE_HPT_RESIZE)) return -ENODEV; - printk(KERN_INFO "lpar: Attempting to resize HPT to shift %lu\n", - shift); + pr_info("Attempting to resize HPT to shift %lu\n", shift); t0 = ktime_get(); @@ -672,8 +675,7 @@ static int pseries_lpar_resize_hpt(unsigned long shift) /* prepare with shift==0 cancels an in-progress resize */ rc = plpar_resize_hpt_prepare(0, 0); if (rc != H_SUCCESS) - printk(KERN_WARNING - "lpar: Unexpected error %d cancelling timed out HPT resize\n", + pr_warn("Unexpected error %d cancelling timed out HPT resize\n", rc); return -ETIMEDOUT; } @@ -691,9 +693,7 @@ static int pseries_lpar_resize_hpt(unsigned long shift) case H_RESOURCE: return -EPERM; default: - printk(KERN_WARNING - "lpar: Unexpected error %d from H_RESIZE_HPT_PREPARE\n", - rc); + pr_warn("Unexpected error %d from H_RESIZE_HPT_PREPARE\n", rc); return -EIO; } @@ -706,22 +706,19 @@ static int pseries_lpar_resize_hpt(unsigned long shift) if (rc != 0) { switch (state.commit_rc) { case H_PTEG_FULL: - printk(KERN_WARNING - "lpar: Hash collision while resizing HPT\n"); + pr_warn("Hash collision while resizing HPT\n"); return -ENOSPC; default: - printk(KERN_WARNING - "lpar: Unexpected error %d from H_RESIZE_HPT_COMMIT\n", - state.commit_rc); + pr_warn("Unexpected error %d from H_RESIZE_HPT_COMMIT\n", + state.commit_rc); return -EIO; }; } - printk(KERN_INFO - "lpar: HPT resize to shift %lu complete (%lld ms / %lld ms)\n", - shift, (long long) ktime_ms_delta(t1, t0), - (long long) ktime_ms_delta(t2, t1)); + pr_info("HPT resize to shift %lu complete (%lld ms / %lld ms)\n", + shift, (long long) ktime_ms_delta(t1, t0), + (long long) ktime_ms_delta(t2, t1)); return 0; } @@ -785,13 +782,13 @@ static int __init cmo_free_hint(char *str) parm = strstrip(str); if (strcasecmp(parm, "no") == 0 || strcasecmp(parm, "off") == 0) { - printk(KERN_INFO "cmo_free_hint: CMO free page hinting is not active.\n"); + pr_info("%s: CMO free page hinting is not active.\n", __func__); cmo_free_hint_flag = 0; return 1; } cmo_free_hint_flag = 1; - printk(KERN_INFO "cmo_free_hint: CMO free page hinting is active.\n"); + pr_info("%s: CMO free page hinting is active.\n", __func__); if (strcasecmp(parm, "yes") == 0 || strcasecmp(parm, "on") == 0) return 1; diff --git a/arch/powerpc/platforms/pseries/mobility.c b/arch/powerpc/platforms/pseries/mobility.c index 8a8033a249c7..f0e30dc94988 100644 --- a/arch/powerpc/platforms/pseries/mobility.c +++ b/arch/powerpc/platforms/pseries/mobility.c @@ -17,6 +17,7 @@ #include <linux/device.h> #include <linux/delay.h> #include <linux/slab.h> +#include <linux/stringify.h> #include <asm/machdep.h> #include <asm/rtas.h> diff --git a/arch/powerpc/platforms/pseries/ras.c b/arch/powerpc/platforms/pseries/ras.c index 5e1ef9150182..851ce326874a 100644 --- a/arch/powerpc/platforms/pseries/ras.c +++ b/arch/powerpc/platforms/pseries/ras.c @@ -22,6 +22,7 @@ #include <linux/of.h> #include <linux/fs.h> #include <linux/reboot.h> +#include <linux/irq_work.h> #include <asm/machdep.h> #include <asm/rtas.h> @@ -32,11 +33,13 @@ static unsigned char ras_log_buf[RTAS_ERROR_LOG_MAX]; static DEFINE_SPINLOCK(ras_log_buf_lock); -static char global_mce_data_buf[RTAS_ERROR_LOG_MAX]; -static DEFINE_PER_CPU(__u64, mce_data_buf); - static int ras_check_exception_token; +static void mce_process_errlog_event(struct irq_work *work); +static struct irq_work mce_errlog_process_work = { + .func = mce_process_errlog_event, +}; + #define EPOW_SENSOR_TOKEN 9 #define EPOW_SENSOR_INDEX 0 @@ -330,16 +333,20 @@ static irqreturn_t ras_error_interrupt(int irq, void *dev_id) ((((A) >= 0x7000) && ((A) < 0x7ff0)) || \ (((A) >= rtas.base) && ((A) < (rtas.base + rtas.size - 16)))) +static inline struct rtas_error_log *fwnmi_get_errlog(void) +{ + return (struct rtas_error_log *)local_paca->mce_data_buf; +} + /* * Get the error information for errors coming through the * FWNMI vectors. The pt_regs' r3 will be updated to reflect * the actual r3 if possible, and a ptr to the error log entry * will be returned if found. * - * If the RTAS error is not of the extended type, then we put it in a per - * cpu 64bit buffer. If it is the extended type we use global_mce_data_buf. + * Use one buffer mce_data_buf per cpu to store RTAS error. * - * The global_mce_data_buf does not have any locks or protection around it, + * The mce_data_buf does not have any locks or protection around it, * if a second machine check comes in, or a system reset is done * before we have logged the error, then we will get corruption in the * error log. This is preferable over holding off on calling @@ -349,7 +356,7 @@ static irqreturn_t ras_error_interrupt(int irq, void *dev_id) static struct rtas_error_log *fwnmi_get_errinfo(struct pt_regs *regs) { unsigned long *savep; - struct rtas_error_log *h, *errhdr = NULL; + struct rtas_error_log *h; /* Mask top two bits */ regs->gpr[3] &= ~(0x3UL << 62); @@ -360,24 +367,22 @@ static struct rtas_error_log *fwnmi_get_errinfo(struct pt_regs *regs) } savep = __va(regs->gpr[3]); - regs->gpr[3] = savep[0]; /* restore original r3 */ + regs->gpr[3] = be64_to_cpu(savep[0]); /* restore original r3 */ - /* If it isn't an extended log we can use the per cpu 64bit buffer */ h = (struct rtas_error_log *)&savep[1]; + /* Use the per cpu buffer from paca to store rtas error log */ + memset(local_paca->mce_data_buf, 0, RTAS_ERROR_LOG_MAX); if (!rtas_error_extended(h)) { - memcpy(this_cpu_ptr(&mce_data_buf), h, sizeof(__u64)); - errhdr = (struct rtas_error_log *)this_cpu_ptr(&mce_data_buf); + memcpy(local_paca->mce_data_buf, h, sizeof(__u64)); } else { int len, error_log_length; error_log_length = 8 + rtas_error_extended_log_length(h); - len = max_t(int, error_log_length, RTAS_ERROR_LOG_MAX); - memset(global_mce_data_buf, 0, RTAS_ERROR_LOG_MAX); - memcpy(global_mce_data_buf, h, len); - errhdr = (struct rtas_error_log *)global_mce_data_buf; + len = min_t(int, error_log_length, RTAS_ERROR_LOG_MAX); + memcpy(local_paca->mce_data_buf, h, len); } - return errhdr; + return (struct rtas_error_log *)local_paca->mce_data_buf; } /* Call this when done with the data returned by FWNMI_get_errinfo. @@ -423,6 +428,17 @@ int pSeries_system_reset_exception(struct pt_regs *regs) } /* + * Process MCE rtas errlog event. + */ +static void mce_process_errlog_event(struct irq_work *work) +{ + struct rtas_error_log *err; + + err = fwnmi_get_errlog(); + log_error((char *)err, ERR_TYPE_RTAS_LOG, 0); +} + +/* * See if we can recover from a machine check exception. * This is only called on power4 (or above) and only via * the Firmware Non-Maskable Interrupts (fwnmi) handler @@ -466,7 +482,8 @@ static int recover_mce(struct pt_regs *regs, struct rtas_error_log *err) recovered = 1; } - log_error((char *)err, ERR_TYPE_RTAS_LOG, 0); + /* Queue irq work to log this rtas event later. */ + irq_work_queue(&mce_errlog_process_work); return recovered; } diff --git a/arch/powerpc/platforms/pseries/setup.c b/arch/powerpc/platforms/pseries/setup.c index 8a4868a3964b..ba1791fd3234 100644 --- a/arch/powerpc/platforms/pseries/setup.c +++ b/arch/powerpc/platforms/pseries/setup.c @@ -41,6 +41,7 @@ #include <linux/root_dev.h> #include <linux/of.h> #include <linux/of_pci.h> +#include <linux/memblock.h> #include <asm/mmu.h> #include <asm/processor.h> @@ -69,6 +70,7 @@ #include <asm/kexec.h> #include <asm/isa-bridge.h> #include <asm/security_features.h> +#include <asm/asm-const.h> #include "pseries.h" #include "../../../../drivers/pci/pci.h" @@ -102,6 +104,9 @@ static void pSeries_show_cpuinfo(struct seq_file *m) static void __init fwnmi_init(void) { unsigned long system_reset_addr, machine_check_addr; + u8 *mce_data_buf; + unsigned int i; + int nr_cpus = num_possible_cpus(); int ibm_nmi_register = rtas_token("ibm,nmi-register"); if (ibm_nmi_register == RTAS_UNKNOWN_SERVICE) @@ -115,6 +120,18 @@ static void __init fwnmi_init(void) if (0 == rtas_call(ibm_nmi_register, 2, 1, NULL, system_reset_addr, machine_check_addr)) fwnmi_active = 1; + + /* + * Allocate a chunk for per cpu buffer to hold rtas errorlog. + * It will be used in real mode mce handler, hence it needs to be + * below RMA. + */ + mce_data_buf = __va(memblock_alloc_base(RTAS_ERROR_LOG_MAX * nr_cpus, + RTAS_ERROR_LOG_MAX, ppc64_rma_size)); + for_each_possible_cpu(i) { + paca_ptrs[i]->mce_data_buf = mce_data_buf + + (RTAS_ERROR_LOG_MAX * i); + } } static void pseries_8259_cascade(struct irq_desc *desc) @@ -485,6 +502,12 @@ static void init_cpu_char_feature_flags(struct h_cpu_char_result *result) if (result->character & H_CPU_CHAR_COUNT_CACHE_DISABLED) security_ftr_set(SEC_FTR_COUNT_CACHE_DISABLED); + if (result->character & H_CPU_CHAR_BCCTR_FLUSH_ASSIST) + security_ftr_set(SEC_FTR_BCCTR_FLUSH_ASSIST); + + if (result->behaviour & H_CPU_BEHAV_FLUSH_COUNT_CACHE) + security_ftr_set(SEC_FTR_FLUSH_COUNT_CACHE); + /* * The features below are enabled by default, so we instead look to see * if firmware has *disabled* them, and clear them if so. @@ -535,7 +558,7 @@ void pseries_setup_rfi_flush(void) security_ftr_enabled(SEC_FTR_L1D_FLUSH_PR); setup_rfi_flush(types, enable); - setup_barrier_nospec(); + setup_count_cache_flush(); } #ifdef CONFIG_PCI_IOV @@ -647,6 +670,15 @@ void of_pci_parse_iov_addrs(struct pci_dev *dev, const int *indexes) } } +static void pseries_disable_sriov_resources(struct pci_dev *pdev) +{ + int i; + + pci_warn(pdev, "No hypervisor support for SR-IOV on this device, IOV BARs disabled.\n"); + for (i = 0; i < PCI_SRIOV_NUM_BARS; i++) + pdev->resource[i + PCI_IOV_RESOURCES].flags = 0; +} + static void pseries_pci_fixup_resources(struct pci_dev *pdev) { const int *indexes; @@ -654,10 +686,10 @@ static void pseries_pci_fixup_resources(struct pci_dev *pdev) /*Firmware must support open sriov otherwise dont configure*/ indexes = of_get_property(dn, "ibm,open-sriov-vf-bar-info", NULL); - if (!indexes) - return; - /* Assign the addresses from device tree*/ - of_pci_set_vf_bar_size(pdev, indexes); + if (indexes) + of_pci_set_vf_bar_size(pdev, indexes); + else + pseries_disable_sriov_resources(pdev); } static void pseries_pci_fixup_iov_resources(struct pci_dev *pdev) @@ -669,10 +701,10 @@ static void pseries_pci_fixup_iov_resources(struct pci_dev *pdev) return; /*Firmware must support open sriov otherwise dont configure*/ indexes = of_get_property(dn, "ibm,open-sriov-vf-bar-info", NULL); - if (!indexes) - return; - /* Assign the addresses from device tree*/ - of_pci_parse_iov_addrs(pdev, indexes); + if (indexes) + of_pci_parse_iov_addrs(pdev, indexes); + else + pseries_disable_sriov_resources(pdev); } static resource_size_t pseries_pci_iov_resource_alignment(struct pci_dev *pdev, diff --git a/arch/powerpc/purgatory/trampoline.S b/arch/powerpc/purgatory/trampoline.S index 4aad9dd10ace..1e1129553fd7 100644 --- a/arch/powerpc/purgatory/trampoline.S +++ b/arch/powerpc/purgatory/trampoline.S @@ -12,15 +12,7 @@ * Software Foundation (version 2 of the License). */ -#if defined(__LITTLE_ENDIAN__) -#define STWX_BE stwbrx -#define LWZX_BE lwbrx -#elif defined(__BIG_ENDIAN__) -#define STWX_BE stwx -#define LWZX_BE lwzx -#else -#error no endianness defined! -#endif +#include <asm/asm-compat.h> .machine ppc64 .balign 256 diff --git a/arch/powerpc/sysdev/Makefile b/arch/powerpc/sysdev/Makefile index ea2f595b5133..f730539074c4 100644 --- a/arch/powerpc/sysdev/Makefile +++ b/arch/powerpc/sysdev/Makefile @@ -48,7 +48,7 @@ obj-$(CONFIG_UCODE_PATCH) += micropatch.o obj-$(CONFIG_PPC_MPC512x) += mpc5xxx_clocks.o obj-$(CONFIG_PPC_MPC52xx) += mpc5xxx_clocks.o -ifeq ($(CONFIG_SUSPEND),y) +ifdef CONFIG_SUSPEND obj-$(CONFIG_6xx) += 6xx-suspend.o endif diff --git a/arch/powerpc/sysdev/cpm1.c b/arch/powerpc/sysdev/cpm1.c index 5240d3a74a10..4f8dcf124828 100644 --- a/arch/powerpc/sysdev/cpm1.c +++ b/arch/powerpc/sysdev/cpm1.c @@ -38,7 +38,6 @@ #include <asm/8xx_immap.h> #include <asm/cpm1.h> #include <asm/io.h> -#include <asm/tlbflush.h> #include <asm/rheap.h> #include <asm/prom.h> #include <asm/cpm.h> diff --git a/arch/powerpc/sysdev/fsl_mpic_err.c b/arch/powerpc/sysdev/fsl_mpic_err.c index 488ec453038a..2a98837dc6ba 100644 --- a/arch/powerpc/sysdev/fsl_mpic_err.c +++ b/arch/powerpc/sysdev/fsl_mpic_err.c @@ -76,7 +76,7 @@ int mpic_setup_error_int(struct mpic *mpic, int intvec) mpic->flags |= MPIC_FSL_HAS_EIMR; /* allocate interrupt vectors for error interrupts */ for (i = MPIC_MAX_ERR - 1; i >= 0; i--) - mpic->err_int_vecs[i] = --intvec; + mpic->err_int_vecs[i] = intvec--; return 0; } diff --git a/arch/powerpc/sysdev/mpic.c b/arch/powerpc/sysdev/mpic.c index 353b43972bbf..934a77324f6b 100644 --- a/arch/powerpc/sysdev/mpic.c +++ b/arch/powerpc/sysdev/mpic.c @@ -1380,12 +1380,12 @@ struct mpic * __init mpic_alloc(struct device_node *node, * global vector number space, as in case of ipis * and timer interrupts. * - * Available vector space = intvec_top - 12, where 12 + * Available vector space = intvec_top - 13, where 13 * is the number of vectors which have been consumed by - * ipis and timer interrupts. + * ipis, timer interrupts and spurious. */ if (fsl_version >= 0x401) { - ret = mpic_setup_error_int(mpic, intvec_top - 12); + ret = mpic_setup_error_int(mpic, intvec_top - 13); if (ret) return NULL; } diff --git a/arch/powerpc/sysdev/mpic_msgr.c b/arch/powerpc/sysdev/mpic_msgr.c index eb69a5186243..280e964e1aa8 100644 --- a/arch/powerpc/sysdev/mpic_msgr.c +++ b/arch/powerpc/sysdev/mpic_msgr.c @@ -196,7 +196,7 @@ static int mpic_msgr_probe(struct platform_device *dev) /* IO map the message register block. */ of_address_to_resource(np, 0, &rsrc); - msgr_block_addr = ioremap(rsrc.start, rsrc.end - rsrc.start); + msgr_block_addr = ioremap(rsrc.start, resource_size(&rsrc)); if (!msgr_block_addr) { dev_err(&dev->dev, "Failed to iomap MPIC message registers"); return -EFAULT; diff --git a/arch/powerpc/sysdev/msi_bitmap.c b/arch/powerpc/sysdev/msi_bitmap.c index 6243a7e537d0..e64a411d1a00 100644 --- a/arch/powerpc/sysdev/msi_bitmap.c +++ b/arch/powerpc/sysdev/msi_bitmap.c @@ -225,22 +225,23 @@ static void __init test_of_node(void) struct device_node of_node; struct property prop; struct msi_bitmap bmp; - int size = 256; - DECLARE_BITMAP(expected, size); +#define SIZE_EXPECTED 256 + DECLARE_BITMAP(expected, SIZE_EXPECTED); /* There should really be a struct device_node allocator */ memset(&of_node, 0, sizeof(of_node)); of_node_init(&of_node); of_node.full_name = node_name; - WARN_ON(msi_bitmap_alloc(&bmp, size, &of_node)); + WARN_ON(msi_bitmap_alloc(&bmp, SIZE_EXPECTED, &of_node)); /* No msi-available-ranges, so expect > 0 */ WARN_ON(msi_bitmap_reserve_dt_hwirqs(&bmp) <= 0); /* Should all still be free */ - WARN_ON(bitmap_find_free_region(bmp.bitmap, size, get_count_order(size))); - bitmap_release_region(bmp.bitmap, 0, get_count_order(size)); + WARN_ON(bitmap_find_free_region(bmp.bitmap, SIZE_EXPECTED, + get_count_order(SIZE_EXPECTED))); + bitmap_release_region(bmp.bitmap, 0, get_count_order(SIZE_EXPECTED)); /* Now create a fake msi-available-ranges property */ @@ -256,8 +257,8 @@ static void __init test_of_node(void) WARN_ON(msi_bitmap_reserve_dt_hwirqs(&bmp)); /* Check we got the expected result */ - WARN_ON(bitmap_parselist(expected_str, expected, size)); - WARN_ON(!bitmap_equal(expected, bmp.bitmap, size)); + WARN_ON(bitmap_parselist(expected_str, expected, SIZE_EXPECTED)); + WARN_ON(!bitmap_equal(expected, bmp.bitmap, SIZE_EXPECTED)); msi_bitmap_free(&bmp); kfree(bmp.bitmap); diff --git a/arch/powerpc/sysdev/xive/common.c b/arch/powerpc/sysdev/xive/common.c index 3459015092fa..e8f5b0551095 100644 --- a/arch/powerpc/sysdev/xive/common.c +++ b/arch/powerpc/sysdev/xive/common.c @@ -319,7 +319,7 @@ void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd) * The FW told us to call it. This happens for some * interrupt sources that need additional HW whacking * beyond the ESB manipulation. For example LPC interrupts - * on P9 DD1.0 need a latch to be clared in the LPC bridge + * on P9 DD1.0 needed a latch to be clared in the LPC bridge * itself. The Firmware will take care of it. */ if (WARN_ON_ONCE(!xive_ops->eoi)) @@ -337,9 +337,9 @@ void xive_do_source_eoi(u32 hw_irq, struct xive_irq_data *xd) * This allows us to then do a re-trigger if Q was set * rather than synthesizing an interrupt in software * - * For LSIs, using the HW EOI cycle works around a problem - * on P9 DD1 PHBs where the other ESB accesses don't work - * properly. + * For LSIs the HW EOI cycle is used rather than PQ bits, + * as they are automatically re-triggred in HW when still + * pending. */ if (xd->flags & XIVE_IRQ_FLAG_LSI) xive_esb_read(xd, XIVE_ESB_LOAD_EOI); @@ -1408,28 +1408,6 @@ void xive_teardown_cpu(void) xive_cleanup_cpu_queues(cpu, xc); } -void xive_kexec_teardown_cpu(int secondary) -{ - struct xive_cpu *xc = __this_cpu_read(xive_cpu); - unsigned int cpu = smp_processor_id(); - - /* Set CPPR to 0 to disable flow of interrupts */ - xc->cppr = 0; - out_8(xive_tima + xive_tima_offset + TM_CPPR, 0); - - /* Backend cleanup if any */ - if (xive_ops->teardown_cpu) - xive_ops->teardown_cpu(cpu, xc); - -#ifdef CONFIG_SMP - /* Get rid of IPI */ - xive_cleanup_cpu_ipi(cpu, xc); -#endif - - /* Disable and free the queues */ - xive_cleanup_cpu_queues(cpu, xc); -} - void xive_shutdown(void) { xive_ops->shutdown(); diff --git a/arch/powerpc/sysdev/xive/native.c b/arch/powerpc/sysdev/xive/native.c index 311185b9960a..5b20a678d755 100644 --- a/arch/powerpc/sysdev/xive/native.c +++ b/arch/powerpc/sysdev/xive/native.c @@ -109,7 +109,7 @@ int xive_native_configure_irq(u32 hw_irq, u32 target, u8 prio, u32 sw_irq) rc = opal_xive_set_irq_config(hw_irq, target, prio, sw_irq); if (rc != OPAL_BUSY) break; - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); } return rc == 0 ? 0 : -ENXIO; } @@ -163,7 +163,7 @@ int xive_native_configure_queue(u32 vp_id, struct xive_q *q, u8 prio, rc = opal_xive_set_queue_info(vp_id, prio, qpage_phys, order, flags); if (rc != OPAL_BUSY) break; - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); } if (rc) { pr_err("Error %lld setting queue for prio %d\n", rc, prio); @@ -190,7 +190,7 @@ static void __xive_native_disable_queue(u32 vp_id, struct xive_q *q, u8 prio) rc = opal_xive_set_queue_info(vp_id, prio, 0, 0, 0); if (rc != OPAL_BUSY) break; - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); } if (rc) pr_err("Error %lld disabling queue for prio %d\n", rc, prio); @@ -253,7 +253,7 @@ static int xive_native_get_ipi(unsigned int cpu, struct xive_cpu *xc) for (;;) { irq = opal_xive_allocate_irq(chip_id); if (irq == OPAL_BUSY) { - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); continue; } if (irq < 0) { @@ -275,7 +275,7 @@ u32 xive_native_alloc_irq(void) rc = opal_xive_allocate_irq(OPAL_XIVE_ANY_CHIP); if (rc != OPAL_BUSY) break; - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); } if (rc < 0) return 0; @@ -289,7 +289,7 @@ void xive_native_free_irq(u32 irq) s64 rc = opal_xive_free_irq(irq); if (rc != OPAL_BUSY) break; - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); } } EXPORT_SYMBOL_GPL(xive_native_free_irq); @@ -305,7 +305,7 @@ static void xive_native_put_ipi(unsigned int cpu, struct xive_cpu *xc) for (;;) { rc = opal_xive_free_irq(xc->hw_ipi); if (rc == OPAL_BUSY) { - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); continue; } xc->hw_ipi = 0; @@ -395,12 +395,11 @@ static void xive_native_setup_cpu(unsigned int cpu, struct xive_cpu *xc) /* Enable the pool VP */ vp = xive_pool_vps + cpu; - pr_debug("CPU %d setting up pool VP 0x%x\n", cpu, vp); for (;;) { rc = opal_xive_set_vp_info(vp, OPAL_XIVE_VP_ENABLED, 0); if (rc != OPAL_BUSY) break; - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); } if (rc) { pr_err("Failed to enable pool VP on CPU %d\n", cpu); @@ -415,16 +414,9 @@ static void xive_native_setup_cpu(unsigned int cpu, struct xive_cpu *xc) } vp_cam = be64_to_cpu(vp_cam_be); - pr_debug("VP CAM = %llx\n", vp_cam); - /* Push it on the CPU (set LSMFB to 0xff to skip backlog scan) */ - pr_debug("(Old HW value: %08x)\n", - in_be32(xive_tima + TM_QW2_HV_POOL + TM_WORD2)); out_be32(xive_tima + TM_QW2_HV_POOL + TM_WORD0, 0xff); - out_be32(xive_tima + TM_QW2_HV_POOL + TM_WORD2, - TM_QW2W2_VP | vp_cam); - pr_debug("(New HW value: %08x)\n", - in_be32(xive_tima + TM_QW2_HV_POOL + TM_WORD2)); + out_be32(xive_tima + TM_QW2_HV_POOL + TM_WORD2, TM_QW2W2_VP | vp_cam); } static void xive_native_teardown_cpu(unsigned int cpu, struct xive_cpu *xc) @@ -444,7 +436,7 @@ static void xive_native_teardown_cpu(unsigned int cpu, struct xive_cpu *xc) rc = opal_xive_set_vp_info(vp, 0, 0); if (rc != OPAL_BUSY) break; - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); } } @@ -645,7 +637,7 @@ u32 xive_native_alloc_vp_block(u32 max_vcpus) rc = opal_xive_alloc_vp_block(order); switch (rc) { case OPAL_BUSY: - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); break; case OPAL_XIVE_PROVISIONING: if (!xive_native_provision_pages()) @@ -687,7 +679,7 @@ int xive_native_enable_vp(u32 vp_id, bool single_escalation) rc = opal_xive_set_vp_info(vp_id, flags, 0); if (rc != OPAL_BUSY) break; - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); } return rc ? -EIO : 0; } @@ -701,7 +693,7 @@ int xive_native_disable_vp(u32 vp_id) rc = opal_xive_set_vp_info(vp_id, 0, 0); if (rc != OPAL_BUSY) break; - msleep(1); + msleep(OPAL_BUSY_DELAY_MS); } return rc ? -EIO : 0; } diff --git a/arch/powerpc/tools/checkpatch.sh b/arch/powerpc/tools/checkpatch.sh new file mode 100755 index 000000000000..1fad3fb90e7c --- /dev/null +++ b/arch/powerpc/tools/checkpatch.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# SPDX-License-Identifier: GPL-2.0+ +# Copyright 2018, Michael Ellerman, IBM Corporation. +# +# Wrapper around checkpatch that uses our preferred settings + +script_base=$(realpath $(dirname $0)) + +exec $script_base/../../../scripts/checkpatch.pl \ + --subjective \ + --no-summary \ + --max-line-length=90 \ + --show-types \ + --ignore ARCH_INCLUDE_LINUX \ + --ignore BIT_MACRO \ + --ignore COMPARISON_TO_NULL \ + --ignore EMAIL_SUBJECT \ + --ignore FILE_PATH_CHANGES \ + --ignore GLOBAL_INITIALISERS \ + --ignore LINE_SPACING \ + --ignore MULTIPLE_ASSIGNMENTS \ + $@ diff --git a/arch/powerpc/xmon/spr_access.S b/arch/powerpc/xmon/spr_access.S index 4099cbcddaaa..720a52afdd58 100644 --- a/arch/powerpc/xmon/spr_access.S +++ b/arch/powerpc/xmon/spr_access.S @@ -1,5 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ #include <asm/ppc_asm.h> +#include <asm/asm-compat.h> /* unsigned long xmon_mfspr(sprn, default_value) */ _GLOBAL(xmon_mfspr) diff --git a/arch/powerpc/xmon/xmon.c b/arch/powerpc/xmon/xmon.c index 196978733e64..4264aedc7775 100644 --- a/arch/powerpc/xmon/xmon.c +++ b/arch/powerpc/xmon/xmon.c @@ -56,6 +56,7 @@ #include <asm/opal.h> #include <asm/firmware.h> #include <asm/code-patching.h> +#include <asm/sections.h> #ifdef CONFIG_PPC64 #include <asm/hvcall.h> @@ -244,6 +245,7 @@ Commands:\n\ f flush cache\n\ la lookup symbol+offset of specified address\n\ ls lookup address of specified symbol\n\ + lp s [#] lookup address of percpu symbol s for current cpu, or cpu #\n\ m examine/change memory\n\ mm move a block of memory\n\ ms set a block of memory\n\ @@ -918,13 +920,13 @@ static void remove_cpu_bpts(void) static void show_uptime(void) { - struct timespec uptime; + struct timespec64 uptime; if (setjmp(bus_error_jmp) == 0) { catch_memory_errors = 1; sync(); - get_monotonic_boottime(&uptime); + ktime_get_coarse_boottime_ts64(&uptime); printf("Uptime: %lu.%.2lu seconds\n", (unsigned long)uptime.tv_sec, ((unsigned long)uptime.tv_nsec / (NSEC_PER_SEC/100))); @@ -2429,7 +2431,6 @@ static void dump_one_paca(int cpu) DUMP(p, thread_idle_state, "%#-*x"); DUMP(p, thread_mask, "%#-*x"); DUMP(p, subcore_sibling_mask, "%#-*x"); - DUMP(p, thread_sibling_pacas, "%-*px"); DUMP(p, requested_psscr, "%#-*llx"); DUMP(p, stop_sprs.pid, "%#-*llx"); DUMP(p, stop_sprs.ldbar, "%#-*llx"); @@ -3353,7 +3354,8 @@ static void symbol_lookup(void) { int type = inchar(); - unsigned long addr; + unsigned long addr, cpu; + void __percpu *ptr = NULL; static char tmp[64]; switch (type) { @@ -3377,6 +3379,34 @@ symbol_lookup(void) catch_memory_errors = 0; termch = 0; break; + case 'p': + getstring(tmp, 64); + if (setjmp(bus_error_jmp) == 0) { + catch_memory_errors = 1; + sync(); + ptr = (void __percpu *)kallsyms_lookup_name(tmp); + sync(); + } + + if (ptr && + ptr >= (void __percpu *)__per_cpu_start && + ptr < (void __percpu *)__per_cpu_end) + { + if (scanhex(&cpu) && cpu < num_possible_cpus()) { + addr = (unsigned long)per_cpu_ptr(ptr, cpu); + } else { + cpu = raw_smp_processor_id(); + addr = (unsigned long)this_cpu_ptr(ptr); + } + + printf("%s for cpu 0x%lx: %lx\n", tmp, cpu, addr); + } else { + printf("Percpu symbol '%s' not found.\n", tmp); + } + + catch_memory_errors = 0; + termch = 0; + break; } } diff --git a/arch/x86/include/asm/pgtable-invert.h b/arch/x86/include/asm/pgtable-invert.h index 44b1203ece12..a0c1525f1b6f 100644 --- a/arch/x86/include/asm/pgtable-invert.h +++ b/arch/x86/include/asm/pgtable-invert.h @@ -4,9 +4,18 @@ #ifndef __ASSEMBLY__ +/* + * A clear pte value is special, and doesn't get inverted. + * + * Note that even users that only pass a pgprot_t (rather + * than a full pte) won't trigger the special zero case, + * because even PAGE_NONE has _PAGE_PROTNONE | _PAGE_ACCESSED + * set. So the all zero case really is limited to just the + * cleared page table entry case. + */ static inline bool __pte_needs_invert(u64 val) { - return !(val & _PAGE_PRESENT); + return val && !(val & _PAGE_PRESENT); } /* Get a mask to xor with the page table entry to get the correct pfn. */ diff --git a/drivers/bus/imx-weim.c b/drivers/bus/imx-weim.c index 3d56ebcda720..6a94aa6a22c2 100644 --- a/drivers/bus/imx-weim.c +++ b/drivers/bus/imx-weim.c @@ -45,6 +45,8 @@ static const struct imx_weim_devtype imx51_weim_devtype = { .cs_stride = 0x18, }; +#define MAX_CS_REGS_COUNT 6 + static const struct of_device_id weim_id_table[] = { /* i.MX1/21 */ { .compatible = "fsl,imx1-weim", .data = &imx1_weim_devtype, }, @@ -112,9 +114,12 @@ err: static int __init weim_timing_setup(struct device_node *np, void __iomem *base, const struct imx_weim_devtype *devtype) { - u32 cs_idx, value[devtype->cs_regs_count]; + u32 cs_idx, value[MAX_CS_REGS_COUNT]; int i, ret; + if (WARN_ON(devtype->cs_regs_count > MAX_CS_REGS_COUNT)) + return -EINVAL; + /* get the CS index from this child node's "reg" property. */ ret = of_property_read_u32(np, "reg", &cs_idx); if (ret) diff --git a/drivers/cpufreq/powernv-cpufreq.c b/drivers/cpufreq/powernv-cpufreq.c index 54edaec1e608..bf6519cf64bc 100644 --- a/drivers/cpufreq/powernv-cpufreq.c +++ b/drivers/cpufreq/powernv-cpufreq.c @@ -758,8 +758,13 @@ static int powernv_cpufreq_target_index(struct cpufreq_policy *policy, cur_msec = jiffies_to_msecs(get_jiffies_64()); - spin_lock(&gpstates->gpstate_lock); freq_data.pstate_id = idx_to_pstate(new_index); + if (!gpstates) { + freq_data.gpstate_id = freq_data.pstate_id; + goto no_gpstate; + } + + spin_lock(&gpstates->gpstate_lock); if (!gpstates->last_sampled_time) { gpstate_idx = new_index; @@ -809,6 +814,7 @@ gpstates_done: spin_unlock(&gpstates->gpstate_lock); +no_gpstate: /* * Use smp_call_function to send IPI and execute the * mtspr on target CPU. We could do that without IPI @@ -843,6 +849,13 @@ static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy) kernfs_put(kn); } + policy->freq_table = powernv_freqs; + policy->fast_switch_possible = true; + + if (pvr_version_is(PVR_POWER9)) + return 0; + + /* Initialise Gpstate ramp-down timer only on POWER8 */ gpstates = kzalloc(sizeof(*gpstates), GFP_KERNEL); if (!gpstates) return -ENOMEM; @@ -857,8 +870,6 @@ static int powernv_cpufreq_cpu_init(struct cpufreq_policy *policy) msecs_to_jiffies(GPSTATE_TIMER_INTERVAL); spin_lock_init(&gpstates->gpstate_lock); - policy->freq_table = powernv_freqs; - policy->fast_switch_possible = true; return 0; } @@ -998,7 +1009,8 @@ static void powernv_cpufreq_stop_cpu(struct cpufreq_policy *policy) freq_data.pstate_id = idx_to_pstate(powernv_pstate_info.min); freq_data.gpstate_id = idx_to_pstate(powernv_pstate_info.min); smp_call_function_single(policy->cpu, set_pstate, &freq_data, 1); - del_timer_sync(&gpstates->timer); + if (gpstates) + del_timer_sync(&gpstates->timer); } static unsigned int powernv_fast_switch(struct cpufreq_policy *policy, diff --git a/drivers/cpuidle/cpuidle-powernv.c b/drivers/cpuidle/cpuidle-powernv.c index d29e4f041efe..84b1ebe212b3 100644 --- a/drivers/cpuidle/cpuidle-powernv.c +++ b/drivers/cpuidle/cpuidle-powernv.c @@ -242,6 +242,7 @@ static inline void add_powernv_state(int index, const char *name, powernv_states[index].target_residency = target_residency; powernv_states[index].exit_latency = exit_latency; powernv_states[index].enter = idle_fn; + /* For power8 and below psscr_* will be 0 */ stop_psscr_table[index].val = psscr_val; stop_psscr_table[index].mask = psscr_mask; } @@ -263,186 +264,80 @@ static inline int validate_dt_prop_sizes(const char *prop1, int prop1_len, extern u32 pnv_get_supported_cpuidle_states(void); static int powernv_add_idle_states(void) { - struct device_node *power_mgt; int nr_idle_states = 1; /* Snooze */ - int dt_idle_states, count; - u32 latency_ns[CPUIDLE_STATE_MAX]; - u32 residency_ns[CPUIDLE_STATE_MAX]; - u32 flags[CPUIDLE_STATE_MAX]; - u64 psscr_val[CPUIDLE_STATE_MAX]; - u64 psscr_mask[CPUIDLE_STATE_MAX]; - const char *names[CPUIDLE_STATE_MAX]; + int dt_idle_states; u32 has_stop_states = 0; - int i, rc; + int i; u32 supported_flags = pnv_get_supported_cpuidle_states(); /* Currently we have snooze statically defined */ - - power_mgt = of_find_node_by_path("/ibm,opal/power-mgt"); - if (!power_mgt) { - pr_warn("opal: PowerMgmt Node not found\n"); - goto out; - } - - /* Read values of any property to determine the num of idle states */ - dt_idle_states = of_property_count_u32_elems(power_mgt, "ibm,cpu-idle-state-flags"); - if (dt_idle_states < 0) { - pr_warn("cpuidle-powernv: no idle states found in the DT\n"); + if (nr_pnv_idle_states <= 0) { + pr_warn("cpuidle-powernv : Only Snooze is available\n"); goto out; } - count = of_property_count_u32_elems(power_mgt, - "ibm,cpu-idle-state-latencies-ns"); - - if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", dt_idle_states, - "ibm,cpu-idle-state-latencies-ns", - count) != 0) - goto out; - - count = of_property_count_strings(power_mgt, - "ibm,cpu-idle-state-names"); - if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", dt_idle_states, - "ibm,cpu-idle-state-names", - count) != 0) - goto out; + /* TODO: Count only states which are eligible for cpuidle */ + dt_idle_states = nr_pnv_idle_states; /* * Since snooze is used as first idle state, max idle states allowed is * CPUIDLE_STATE_MAX -1 */ - if (dt_idle_states > CPUIDLE_STATE_MAX - 1) { + if (nr_pnv_idle_states > CPUIDLE_STATE_MAX - 1) { pr_warn("cpuidle-powernv: discovered idle states more than allowed"); dt_idle_states = CPUIDLE_STATE_MAX - 1; } - if (of_property_read_u32_array(power_mgt, - "ibm,cpu-idle-state-flags", flags, dt_idle_states)) { - pr_warn("cpuidle-powernv : missing ibm,cpu-idle-state-flags in DT\n"); - goto out; - } - - if (of_property_read_u32_array(power_mgt, - "ibm,cpu-idle-state-latencies-ns", latency_ns, - dt_idle_states)) { - pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-latencies-ns in DT\n"); - goto out; - } - if (of_property_read_string_array(power_mgt, - "ibm,cpu-idle-state-names", names, dt_idle_states) < 0) { - pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-names in DT\n"); - goto out; - } - /* * If the idle states use stop instruction, probe for psscr values * and psscr mask which are necessary to specify required stop level. */ - has_stop_states = (flags[0] & + has_stop_states = (pnv_idle_states[0].flags & (OPAL_PM_STOP_INST_FAST | OPAL_PM_STOP_INST_DEEP)); - if (has_stop_states) { - count = of_property_count_u64_elems(power_mgt, - "ibm,cpu-idle-state-psscr"); - if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", - dt_idle_states, - "ibm,cpu-idle-state-psscr", - count) != 0) - goto out; - - count = of_property_count_u64_elems(power_mgt, - "ibm,cpu-idle-state-psscr-mask"); - if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", - dt_idle_states, - "ibm,cpu-idle-state-psscr-mask", - count) != 0) - goto out; - - if (of_property_read_u64_array(power_mgt, - "ibm,cpu-idle-state-psscr", psscr_val, dt_idle_states)) { - pr_warn("cpuidle-powernv: missing ibm,cpu-idle-state-psscr in DT\n"); - goto out; - } - - if (of_property_read_u64_array(power_mgt, - "ibm,cpu-idle-state-psscr-mask", - psscr_mask, dt_idle_states)) { - pr_warn("cpuidle-powernv:Missing ibm,cpu-idle-state-psscr-mask in DT\n"); - goto out; - } - } - - count = of_property_count_u32_elems(power_mgt, - "ibm,cpu-idle-state-residency-ns"); - - if (count < 0) { - rc = count; - } else if (validate_dt_prop_sizes("ibm,cpu-idle-state-flags", - dt_idle_states, - "ibm,cpu-idle-state-residency-ns", - count) != 0) { - goto out; - } else { - rc = of_property_read_u32_array(power_mgt, - "ibm,cpu-idle-state-residency-ns", - residency_ns, dt_idle_states); - } for (i = 0; i < dt_idle_states; i++) { unsigned int exit_latency, target_residency; bool stops_timebase = false; + struct pnv_idle_states_t *state = &pnv_idle_states[i]; /* * Skip the platform idle state whose flag isn't in * the supported_cpuidle_states flag mask. */ - if ((flags[i] & supported_flags) != flags[i]) + if ((state->flags & supported_flags) != state->flags) continue; /* * If an idle state has exit latency beyond * POWERNV_THRESHOLD_LATENCY_NS then don't use it * in cpu-idle. */ - if (latency_ns[i] > POWERNV_THRESHOLD_LATENCY_NS) + if (state->latency_ns > POWERNV_THRESHOLD_LATENCY_NS) continue; /* * Firmware passes residency and latency values in ns. * cpuidle expects it in us. */ - exit_latency = DIV_ROUND_UP(latency_ns[i], 1000); - if (!rc) - target_residency = DIV_ROUND_UP(residency_ns[i], 1000); - else - target_residency = 0; - - if (has_stop_states) { - int err = validate_psscr_val_mask(&psscr_val[i], - &psscr_mask[i], - flags[i]); - if (err) { - report_invalid_psscr_val(psscr_val[i], err); + exit_latency = DIV_ROUND_UP(state->latency_ns, 1000); + target_residency = DIV_ROUND_UP(state->residency_ns, 1000); + + if (has_stop_states && !(state->valid)) continue; - } - } - if (flags[i] & OPAL_PM_TIMEBASE_STOP) + if (state->flags & OPAL_PM_TIMEBASE_STOP) stops_timebase = true; - /* - * For nap and fastsleep, use default target_residency - * values if f/w does not expose it. - */ - if (flags[i] & OPAL_PM_NAP_ENABLED) { - if (!rc) - target_residency = 100; + if (state->flags & OPAL_PM_NAP_ENABLED) { /* Add NAP state */ add_powernv_state(nr_idle_states, "Nap", CPUIDLE_FLAG_NONE, nap_loop, target_residency, exit_latency, 0, 0); } else if (has_stop_states && !stops_timebase) { - add_powernv_state(nr_idle_states, names[i], + add_powernv_state(nr_idle_states, state->name, CPUIDLE_FLAG_NONE, stop_loop, target_residency, exit_latency, - psscr_val[i], psscr_mask[i]); + state->psscr_val, + state->psscr_mask); } /* @@ -450,20 +345,19 @@ static int powernv_add_idle_states(void) * within this config dependency check. */ #ifdef CONFIG_TICK_ONESHOT - else if (flags[i] & OPAL_PM_SLEEP_ENABLED || - flags[i] & OPAL_PM_SLEEP_ENABLED_ER1) { - if (!rc) - target_residency = 300000; + else if (state->flags & OPAL_PM_SLEEP_ENABLED || + state->flags & OPAL_PM_SLEEP_ENABLED_ER1) { /* Add FASTSLEEP state */ add_powernv_state(nr_idle_states, "FastSleep", CPUIDLE_FLAG_TIMER_STOP, fastsleep_loop, target_residency, exit_latency, 0, 0); } else if (has_stop_states && stops_timebase) { - add_powernv_state(nr_idle_states, names[i], + add_powernv_state(nr_idle_states, state->name, CPUIDLE_FLAG_TIMER_STOP, stop_loop, target_residency, exit_latency, - psscr_val[i], psscr_mask[i]); + state->psscr_val, + state->psscr_mask); } #endif else diff --git a/drivers/crypto/nx/nx-842-powernv.c b/drivers/crypto/nx/nx-842-powernv.c index 36afd6d8753c..c68df7e8bee1 100644 --- a/drivers/crypto/nx/nx-842-powernv.c +++ b/drivers/crypto/nx/nx-842-powernv.c @@ -24,6 +24,8 @@ #include <asm/icswx.h> #include <asm/vas.h> #include <asm/reg.h> +#include <asm/opal-api.h> +#include <asm/opal.h> MODULE_LICENSE("GPL"); MODULE_AUTHOR("Dan Streetman <ddstreet@ieee.org>"); @@ -753,7 +755,7 @@ static int nx842_open_percpu_txwins(void) } static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, - int vasid) + int vasid, int *ct) { struct vas_window *rxwin = NULL; struct vas_rx_win_attr rxattr; @@ -837,6 +839,15 @@ static int __init vas_cfg_coproc_info(struct device_node *dn, int chip_id, coproc->vas.id = vasid; nx842_add_coprocs_list(coproc, chip_id); + /* + * (lpid, pid, tid) combination has to be unique for each + * coprocessor instance in the system. So to make it + * unique, skiboot uses coprocessor type such as 842 or + * GZIP for pid and provides this value to kernel in pid + * device-tree property. + */ + *ct = pid; + return 0; err_out: @@ -850,6 +861,7 @@ static int __init nx842_powernv_probe_vas(struct device_node *pn) struct device_node *dn; int chip_id, vasid, ret = 0; int nx_fifo_found = 0; + int uninitialized_var(ct); chip_id = of_get_ibm_chip_id(pn); if (chip_id < 0) { @@ -865,7 +877,7 @@ static int __init nx842_powernv_probe_vas(struct device_node *pn) for_each_child_of_node(pn, dn) { if (of_device_is_compatible(dn, "ibm,p9-nx-842")) { - ret = vas_cfg_coproc_info(dn, chip_id, vasid); + ret = vas_cfg_coproc_info(dn, chip_id, vasid, &ct); if (ret) { of_node_put(dn); return ret; @@ -876,9 +888,22 @@ static int __init nx842_powernv_probe_vas(struct device_node *pn) if (!nx_fifo_found) { pr_err("NX842 FIFO nodes are missing\n"); - ret = -EINVAL; + return -EINVAL; } + /* + * Initialize NX instance for both high and normal priority FIFOs. + */ + if (opal_check_token(OPAL_NX_COPROC_INIT)) { + ret = opal_nx_coproc_init(chip_id, ct); + if (ret) { + pr_err("Failed to initialize NX for chip(%d): %d\n", + chip_id, ret); + ret = opal_error_code(ret); + } + } else + pr_warn("Firmware doesn't support NX initialization\n"); + return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c index 9f1a5bd39ae8..5b39d1399630 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_psp.c @@ -131,6 +131,11 @@ psp_cmd_submit_buf(struct psp_context *psp, msleep(1); } + if (ucode) { + ucode->tmr_mc_addr_lo = psp->cmd_buf_mem->resp.fw_addr_lo; + ucode->tmr_mc_addr_hi = psp->cmd_buf_mem->resp.fw_addr_hi; + } + return ret; } diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h index 08e38579af24..bdc472b6e641 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_ucode.h @@ -194,6 +194,7 @@ enum AMDGPU_UCODE_ID { AMDGPU_UCODE_ID_SMC, AMDGPU_UCODE_ID_UVD, AMDGPU_UCODE_ID_VCE, + AMDGPU_UCODE_ID_VCN, AMDGPU_UCODE_ID_MAXIMUM, }; @@ -226,6 +227,9 @@ struct amdgpu_firmware_info { void *kaddr; /* ucode_size_bytes */ uint32_t ucode_size; + /* starting tmr mc address */ + uint32_t tmr_mc_addr_lo; + uint32_t tmr_mc_addr_hi; }; void amdgpu_ucode_print_mc_hdr(const struct common_firmware_header *hdr); diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c index 632fa5980ff4..e5a6db6beab7 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.c @@ -122,8 +122,6 @@ static void amdgpu_uvd_idle_work_handler(struct work_struct *work); int amdgpu_uvd_sw_init(struct amdgpu_device *adev) { - struct amdgpu_ring *ring; - struct drm_sched_rq *rq; unsigned long bo_size; const char *fw_name; const struct common_firmware_header *hdr; @@ -266,13 +264,6 @@ int amdgpu_uvd_sw_init(struct amdgpu_device *adev) } } - ring = &adev->uvd.inst[0].ring; - rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL]; - r = drm_sched_entity_init(&adev->uvd.entity, &rq, 1, NULL); - if (r) { - DRM_ERROR("Failed setting up UVD kernel entity.\n"); - return r; - } for (i = 0; i < adev->uvd.max_handles; ++i) { atomic_set(&adev->uvd.handles[i], 0); adev->uvd.filp[i] = NULL; @@ -311,7 +302,7 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev) for (j = 0; j < adev->uvd.num_uvd_inst; ++j) { if (adev->uvd.harvest_config & (1 << j)) continue; - kfree(adev->uvd.inst[j].saved_bo); + kvfree(adev->uvd.inst[j].saved_bo); amdgpu_bo_free_kernel(&adev->uvd.inst[j].vcpu_bo, &adev->uvd.inst[j].gpu_addr, @@ -327,6 +318,29 @@ int amdgpu_uvd_sw_fini(struct amdgpu_device *adev) return 0; } +/** + * amdgpu_uvd_entity_init - init entity + * + * @adev: amdgpu_device pointer + * + */ +int amdgpu_uvd_entity_init(struct amdgpu_device *adev) +{ + struct amdgpu_ring *ring; + struct drm_sched_rq *rq; + int r; + + ring = &adev->uvd.inst[0].ring; + rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL]; + r = drm_sched_entity_init(&adev->uvd.entity, &rq, 1, NULL); + if (r) { + DRM_ERROR("Failed setting up UVD kernel entity.\n"); + return r; + } + + return 0; +} + int amdgpu_uvd_suspend(struct amdgpu_device *adev) { unsigned size; @@ -354,7 +368,7 @@ int amdgpu_uvd_suspend(struct amdgpu_device *adev) size = amdgpu_bo_size(adev->uvd.inst[j].vcpu_bo); ptr = adev->uvd.inst[j].cpu_addr; - adev->uvd.inst[j].saved_bo = kmalloc(size, GFP_KERNEL); + adev->uvd.inst[j].saved_bo = kvmalloc(size, GFP_KERNEL); if (!adev->uvd.inst[j].saved_bo) return -ENOMEM; @@ -380,7 +394,7 @@ int amdgpu_uvd_resume(struct amdgpu_device *adev) if (adev->uvd.inst[i].saved_bo != NULL) { memcpy_toio(ptr, adev->uvd.inst[i].saved_bo, size); - kfree(adev->uvd.inst[i].saved_bo); + kvfree(adev->uvd.inst[i].saved_bo); adev->uvd.inst[i].saved_bo = NULL; } else { const struct common_firmware_header *hdr; diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h index 33c5f806f925..a3ab1a41060f 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_uvd.h @@ -69,6 +69,7 @@ struct amdgpu_uvd { int amdgpu_uvd_sw_init(struct amdgpu_device *adev); int amdgpu_uvd_sw_fini(struct amdgpu_device *adev); +int amdgpu_uvd_entity_init(struct amdgpu_device *adev); int amdgpu_uvd_suspend(struct amdgpu_device *adev); int amdgpu_uvd_resume(struct amdgpu_device *adev); int amdgpu_uvd_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c index b6ab4f5350c8..0cc5190f4f36 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.c @@ -90,8 +90,6 @@ static void amdgpu_vce_idle_work_handler(struct work_struct *work); */ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size) { - struct amdgpu_ring *ring; - struct drm_sched_rq *rq; const char *fw_name; const struct common_firmware_header *hdr; unsigned ucode_version, version_major, version_minor, binary_id; @@ -188,14 +186,6 @@ int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size) return r; } - ring = &adev->vce.ring[0]; - rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL]; - r = drm_sched_entity_init(&adev->vce.entity, &rq, 1, NULL); - if (r != 0) { - DRM_ERROR("Failed setting up VCE run queue.\n"); - return r; - } - for (i = 0; i < AMDGPU_MAX_VCE_HANDLES; ++i) { atomic_set(&adev->vce.handles[i], 0); adev->vce.filp[i] = NULL; @@ -236,6 +226,29 @@ int amdgpu_vce_sw_fini(struct amdgpu_device *adev) } /** + * amdgpu_vce_entity_init - init entity + * + * @adev: amdgpu_device pointer + * + */ +int amdgpu_vce_entity_init(struct amdgpu_device *adev) +{ + struct amdgpu_ring *ring; + struct drm_sched_rq *rq; + int r; + + ring = &adev->vce.ring[0]; + rq = &ring->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL]; + r = drm_sched_entity_init(&adev->vce.entity, &rq, 1, NULL); + if (r != 0) { + DRM_ERROR("Failed setting up VCE run queue.\n"); + return r; + } + + return 0; +} + +/** * amdgpu_vce_suspend - unpin VCE fw memory * * @adev: amdgpu_device pointer diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h index 71781267ee4c..a1f209eed4c4 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vce.h @@ -55,6 +55,7 @@ struct amdgpu_vce { int amdgpu_vce_sw_init(struct amdgpu_device *adev, unsigned long size); int amdgpu_vce_sw_fini(struct amdgpu_device *adev); +int amdgpu_vce_entity_init(struct amdgpu_device *adev); int amdgpu_vce_suspend(struct amdgpu_device *adev); int amdgpu_vce_resume(struct amdgpu_device *adev); int amdgpu_vce_get_create_msg(struct amdgpu_ring *ring, uint32_t handle, diff --git a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c index 798648a19710..fd654a4406db 100644 --- a/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c +++ b/drivers/gpu/drm/amd/amdgpu/amdgpu_vcn.c @@ -111,9 +111,10 @@ int amdgpu_vcn_sw_init(struct amdgpu_device *adev) version_major, version_minor, family_id); } - bo_size = AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8) - + AMDGPU_VCN_STACK_SIZE + AMDGPU_VCN_HEAP_SIZE + bo_size = AMDGPU_VCN_STACK_SIZE + AMDGPU_VCN_HEAP_SIZE + AMDGPU_VCN_SESSION_SIZE * 40; + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) + bo_size += AMDGPU_GPU_PAGE_ALIGN(le32_to_cpu(hdr->ucode_size_bytes) + 8); r = amdgpu_bo_create_kernel(adev, bo_size, PAGE_SIZE, AMDGPU_GEM_DOMAIN_VRAM, &adev->vcn.vcpu_bo, &adev->vcn.gpu_addr, &adev->vcn.cpu_addr); @@ -129,7 +130,7 @@ int amdgpu_vcn_sw_fini(struct amdgpu_device *adev) { int i; - kfree(adev->vcn.saved_bo); + kvfree(adev->vcn.saved_bo); amdgpu_bo_free_kernel(&adev->vcn.vcpu_bo, &adev->vcn.gpu_addr, @@ -160,7 +161,7 @@ int amdgpu_vcn_suspend(struct amdgpu_device *adev) size = amdgpu_bo_size(adev->vcn.vcpu_bo); ptr = adev->vcn.cpu_addr; - adev->vcn.saved_bo = kmalloc(size, GFP_KERNEL); + adev->vcn.saved_bo = kvmalloc(size, GFP_KERNEL); if (!adev->vcn.saved_bo) return -ENOMEM; @@ -182,18 +183,20 @@ int amdgpu_vcn_resume(struct amdgpu_device *adev) if (adev->vcn.saved_bo != NULL) { memcpy_toio(ptr, adev->vcn.saved_bo, size); - kfree(adev->vcn.saved_bo); + kvfree(adev->vcn.saved_bo); adev->vcn.saved_bo = NULL; } else { const struct common_firmware_header *hdr; unsigned offset; hdr = (const struct common_firmware_header *)adev->vcn.fw->data; - offset = le32_to_cpu(hdr->ucode_array_offset_bytes); - memcpy_toio(adev->vcn.cpu_addr, adev->vcn.fw->data + offset, - le32_to_cpu(hdr->ucode_size_bytes)); - size -= le32_to_cpu(hdr->ucode_size_bytes); - ptr += le32_to_cpu(hdr->ucode_size_bytes); + if (adev->firmware.load_type != AMDGPU_FW_LOAD_PSP) { + offset = le32_to_cpu(hdr->ucode_array_offset_bytes); + memcpy_toio(adev->vcn.cpu_addr, adev->vcn.fw->data + offset, + le32_to_cpu(hdr->ucode_size_bytes)); + size -= le32_to_cpu(hdr->ucode_size_bytes); + ptr += le32_to_cpu(hdr->ucode_size_bytes); + } memset_io(ptr, 0, size); } diff --git a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c index 0ff136d02d9b..02be34e72ed9 100644 --- a/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c +++ b/drivers/gpu/drm/amd/amdgpu/psp_v10_0.c @@ -88,6 +88,9 @@ psp_v10_0_get_fw_type(struct amdgpu_firmware_info *ucode, enum psp_gfx_fw_type * case AMDGPU_UCODE_ID_VCE: *type = GFX_FW_TYPE_VCE; break; + case AMDGPU_UCODE_ID_VCN: + *type = GFX_FW_TYPE_VCN; + break; case AMDGPU_UCODE_ID_MAXIMUM: default: return -EINVAL; diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c index 6fed3d7797a8..8a926d1df939 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v4_2.c @@ -123,6 +123,10 @@ static int uvd_v4_2_sw_init(void *handle) ring = &adev->uvd.inst->ring; sprintf(ring->name, "uvd"); r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst->irq, 0); + if (r) + return r; + + r = amdgpu_uvd_entity_init(adev); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c index aeaa1ca46a99..50248059412e 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v5_0.c @@ -120,6 +120,10 @@ static int uvd_v5_0_sw_init(void *handle) ring = &adev->uvd.inst->ring; sprintf(ring->name, "uvd"); r = amdgpu_ring_init(adev, ring, 512, &adev->uvd.inst->irq, 0); + if (r) + return r; + + r = amdgpu_uvd_entity_init(adev); return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c index 598dbeaba636..6ae82cc2e55e 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v6_0.c @@ -440,6 +440,8 @@ static int uvd_v6_0_sw_init(void *handle) } } + r = amdgpu_uvd_entity_init(adev); + return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c index 5fab3560a71d..9b7f8469bc5c 100644 --- a/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c +++ b/drivers/gpu/drm/amd/amdgpu/uvd_v7_0.c @@ -410,6 +410,7 @@ static int uvd_v7_0_early_init(void *handle) static int uvd_v7_0_sw_init(void *handle) { struct amdgpu_ring *ring; + int i, j, r; struct amdgpu_device *adev = (struct amdgpu_device *)handle; @@ -478,6 +479,10 @@ static int uvd_v7_0_sw_init(void *handle) } } + r = amdgpu_uvd_entity_init(adev); + if (r) + return r; + r = amdgpu_virt_alloc_mm_table(adev); if (r) return r; diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c index d48e877b682e..7eaa54ba016b 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v2_0.c @@ -439,6 +439,8 @@ static int vce_v2_0_sw_init(void *handle) return r; } + r = amdgpu_vce_entity_init(adev); + return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c index cc6ce6cc03f4..c8390f9adfd6 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v3_0.c @@ -448,6 +448,8 @@ static int vce_v3_0_sw_init(void *handle) return r; } + r = amdgpu_vce_entity_init(adev); + return r; } diff --git a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c index 65f8860169e9..2e4d1b5f6243 100644 --- a/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vce_v4_0.c @@ -419,6 +419,7 @@ static int vce_v4_0_sw_init(void *handle) { struct amdgpu_device *adev = (struct amdgpu_device *)handle; struct amdgpu_ring *ring; + unsigned size; int r, i; @@ -438,7 +439,7 @@ static int vce_v4_0_sw_init(void *handle) const struct common_firmware_header *hdr; unsigned size = amdgpu_bo_size(adev->vce.vcpu_bo); - adev->vce.saved_bo = kmalloc(size, GFP_KERNEL); + adev->vce.saved_bo = kvmalloc(size, GFP_KERNEL); if (!adev->vce.saved_bo) return -ENOMEM; @@ -474,6 +475,11 @@ static int vce_v4_0_sw_init(void *handle) return r; } + + r = amdgpu_vce_entity_init(adev); + if (r) + return r; + r = amdgpu_virt_alloc_mm_table(adev); if (r) return r; @@ -490,7 +496,7 @@ static int vce_v4_0_sw_fini(void *handle) amdgpu_virt_free_mm_table(adev); if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { - kfree(adev->vce.saved_bo); + kvfree(adev->vce.saved_bo); adev->vce.saved_bo = NULL; } diff --git a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c index 2ce91a748c40..072371ef5975 100644 --- a/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c +++ b/drivers/gpu/drm/amd/amdgpu/vcn_v1_0.c @@ -100,6 +100,16 @@ static int vcn_v1_0_sw_init(void *handle) if (r) return r; + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + const struct common_firmware_header *hdr; + hdr = (const struct common_firmware_header *)adev->vcn.fw->data; + adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].ucode_id = AMDGPU_UCODE_ID_VCN; + adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].fw = adev->vcn.fw; + adev->firmware.fw_size += + ALIGN(le32_to_cpu(hdr->ucode_size_bytes), PAGE_SIZE); + DRM_INFO("PSP loading VCN firmware\n"); + } + r = amdgpu_vcn_resume(adev); if (r) return r; @@ -265,26 +275,38 @@ static int vcn_v1_0_resume(void *handle) static void vcn_v1_0_mc_resume(struct amdgpu_device *adev) { uint32_t size = AMDGPU_GPU_PAGE_ALIGN(adev->vcn.fw->size + 4); - - WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, + uint32_t offset; + + if (adev->firmware.load_type == AMDGPU_FW_LOAD_PSP) { + WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, + (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_lo)); + WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, + (adev->firmware.ucode[AMDGPU_UCODE_ID_VCN].tmr_mc_addr_hi)); + WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, 0); + offset = 0; + } else { + WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_LOW, lower_32_bits(adev->vcn.gpu_addr)); - WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, + WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE_64BIT_BAR_HIGH, upper_32_bits(adev->vcn.gpu_addr)); - WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, - AMDGPU_UVD_FIRMWARE_OFFSET >> 3); + offset = size; + WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET0, + AMDGPU_UVD_FIRMWARE_OFFSET >> 3); + } + WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE0, size); WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.gpu_addr + size)); + lower_32_bits(adev->vcn.gpu_addr + offset)); WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE1_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.gpu_addr + size)); + upper_32_bits(adev->vcn.gpu_addr + offset)); WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET1, 0); WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE1, AMDGPU_VCN_HEAP_SIZE); WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_LOW, - lower_32_bits(adev->vcn.gpu_addr + size + AMDGPU_VCN_HEAP_SIZE)); + lower_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE)); WREG32_SOC15(UVD, 0, mmUVD_LMI_VCPU_CACHE2_64BIT_BAR_HIGH, - upper_32_bits(adev->vcn.gpu_addr + size + AMDGPU_VCN_HEAP_SIZE)); + upper_32_bits(adev->vcn.gpu_addr + offset + AMDGPU_VCN_HEAP_SIZE)); WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_OFFSET2, 0); WREG32_SOC15(UVD, 0, mmUVD_VCPU_CACHE_SIZE2, AMDGPU_VCN_STACK_SIZE + (AMDGPU_VCN_SESSION_SIZE * 40)); diff --git a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c index 52f2c01349e3..9bfb040352e9 100644 --- a/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c +++ b/drivers/gpu/drm/amd/display/amdgpu_dm/amdgpu_dm_crc.c @@ -98,10 +98,16 @@ int amdgpu_dm_crtc_set_crc_source(struct drm_crtc *crtc, const char *src_name, */ void amdgpu_dm_crtc_handle_crc_irq(struct drm_crtc *crtc) { - struct dm_crtc_state *crtc_state = to_dm_crtc_state(crtc->state); - struct dc_stream_state *stream_state = crtc_state->stream; + struct dm_crtc_state *crtc_state; + struct dc_stream_state *stream_state; uint32_t crcs[3]; + if (crtc == NULL) + return; + + crtc_state = to_dm_crtc_state(crtc->state); + stream_state = crtc_state->stream; + /* Early return if CRC capture is not enabled. */ if (!crtc_state->crc_enabled) return; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_link.c b/drivers/gpu/drm/amd/display/dc/core/dc_link.c index a38e7ad36a7e..326b3e99b7e4 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_link.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_link.c @@ -1812,6 +1812,8 @@ static void enable_link_hdmi(struct pipe_ctx *pipe_ctx) bool is_vga_mode = (stream->timing.h_addressable == 640) && (stream->timing.v_addressable == 480); + if (stream->phy_pix_clk == 0) + stream->phy_pix_clk = stream->timing.pix_clk_khz; if (stream->phy_pix_clk > 340000) is_over_340mhz = true; diff --git a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c index 4ca41d6e3bcf..1644f2a946b0 100644 --- a/drivers/gpu/drm/amd/display/dc/core/dc_resource.c +++ b/drivers/gpu/drm/amd/display/dc/core/dc_resource.c @@ -268,24 +268,30 @@ bool resource_construct( return true; } +static int find_matching_clock_source( + const struct resource_pool *pool, + struct clock_source *clock_source) +{ + + int i; + for (i = 0; i < pool->clk_src_count; i++) { + if (pool->clock_sources[i] == clock_source) + return i; + } + return -1; +} void resource_unreference_clock_source( struct resource_context *res_ctx, const struct resource_pool *pool, struct clock_source *clock_source) { - int i; - - for (i = 0; i < pool->clk_src_count; i++) { - if (pool->clock_sources[i] != clock_source) - continue; + int i = find_matching_clock_source(pool, clock_source); + if (i > -1) res_ctx->clock_source_ref_count[i]--; - break; - } - if (pool->dp_clock_source == clock_source) res_ctx->dp_clock_source_ref_count--; } @@ -295,19 +301,31 @@ void resource_reference_clock_source( const struct resource_pool *pool, struct clock_source *clock_source) { - int i; - for (i = 0; i < pool->clk_src_count; i++) { - if (pool->clock_sources[i] != clock_source) - continue; + int i = find_matching_clock_source(pool, clock_source); + if (i > -1) res_ctx->clock_source_ref_count[i]++; - break; - } if (pool->dp_clock_source == clock_source) res_ctx->dp_clock_source_ref_count++; } +int resource_get_clock_source_reference( + struct resource_context *res_ctx, + const struct resource_pool *pool, + struct clock_source *clock_source) +{ + int i = find_matching_clock_source(pool, clock_source); + + if (i > -1) + return res_ctx->clock_source_ref_count[i]; + + if (pool->dp_clock_source == clock_source) + return res_ctx->dp_clock_source_ref_count; + + return -1; +} + bool resource_are_streams_timing_synchronizable( struct dc_stream_state *stream1, struct dc_stream_state *stream2) @@ -372,11 +390,11 @@ static bool is_sharable_clk_src( return false; if (dc_is_hdmi_signal(pipe_with_clk_src->stream->signal) - && dc_is_dvi_signal(pipe->stream->signal)) + && dc_is_dual_link_signal(pipe->stream->signal)) return false; if (dc_is_hdmi_signal(pipe->stream->signal) - && dc_is_dvi_signal(pipe_with_clk_src->stream->signal)) + && dc_is_dual_link_signal(pipe_with_clk_src->stream->signal)) return false; if (!resource_are_streams_timing_synchronizable( diff --git a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c index 752b3d62e793..eff7d22d78fb 100644 --- a/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c +++ b/drivers/gpu/drm/amd/display/dc/dce/dce_link_encoder.c @@ -930,7 +930,7 @@ void dce110_link_encoder_enable_tmds_output( enum bp_result result; /* Enable the PHY */ - + cntl.connector_obj_id = enc110->base.connector; cntl.action = TRANSMITTER_CONTROL_ENABLE; cntl.engine_id = enc->preferred_engine; cntl.transmitter = enc110->base.transmitter; @@ -972,7 +972,7 @@ void dce110_link_encoder_enable_dp_output( * We need to set number of lanes manually. */ configure_encoder(enc110, link_settings); - + cntl.connector_obj_id = enc110->base.connector; cntl.action = TRANSMITTER_CONTROL_ENABLE; cntl.engine_id = enc->preferred_engine; cntl.transmitter = enc110->base.transmitter; diff --git a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c index 1d98e3678b04..5450d4d38e8a 100644 --- a/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c +++ b/drivers/gpu/drm/amd/display/dc/dce110/dce110_hw_sequencer.c @@ -1908,7 +1908,9 @@ static void dce110_reset_hw_ctx_wrap( pipe_ctx_old->plane_res.mi->funcs->free_mem_input( pipe_ctx_old->plane_res.mi, dc->current_state->stream_count); - if (old_clk) + if (old_clk && 0 == resource_get_clock_source_reference(&context->res_ctx, + dc->res_pool, + old_clk)) old_clk->funcs->cs_power_down(old_clk); dc->hwss.disable_plane(dc, pipe_ctx_old); diff --git a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c index 2ea490f8482e..04b866f0fa1f 100644 --- a/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c +++ b/drivers/gpu/drm/amd/display/dc/dce120/dce120_timing_generator.c @@ -772,7 +772,7 @@ void dce120_tg_set_blank(struct timing_generator *tg, CRTC_REG_SET( CRTC0_CRTC_DOUBLE_BUFFER_CONTROL, - CRTC_BLANK_DATA_DOUBLE_BUFFER_EN, 0); + CRTC_BLANK_DATA_DOUBLE_BUFFER_EN, 1); if (enable_blanking) CRTC_REG_SET(CRTC0_CRTC_BLANK_CONTROL, CRTC_BLANK_DATA_EN, 1); diff --git a/drivers/gpu/drm/amd/display/dc/inc/resource.h b/drivers/gpu/drm/amd/display/dc/inc/resource.h index e92facbd038f..5b321008b0b5 100644 --- a/drivers/gpu/drm/amd/display/dc/inc/resource.h +++ b/drivers/gpu/drm/amd/display/dc/inc/resource.h @@ -103,6 +103,11 @@ void resource_reference_clock_source( const struct resource_pool *pool, struct clock_source *clock_source); +int resource_get_clock_source_reference( + struct resource_context *res_ctx, + const struct resource_pool *pool, + struct clock_source *clock_source); + bool resource_are_streams_timing_synchronizable( struct dc_stream_state *stream1, struct dc_stream_state *stream2); diff --git a/drivers/gpu/drm/i2c/tda9950.c b/drivers/gpu/drm/i2c/tda9950.c index 3f7396caad48..5d2f0d548469 100644 --- a/drivers/gpu/drm/i2c/tda9950.c +++ b/drivers/gpu/drm/i2c/tda9950.c @@ -76,9 +76,12 @@ struct tda9950_priv { static int tda9950_write_range(struct i2c_client *client, u8 addr, u8 *p, int cnt) { struct i2c_msg msg; - u8 buf[cnt + 1]; + u8 buf[CEC_MAX_MSG_SIZE + 3]; int ret; + if (WARN_ON(cnt > sizeof(buf) - 1)) + return -EINVAL; + buf[0] = addr; memcpy(buf + 1, p, cnt); diff --git a/drivers/gpu/drm/i915/gvt/aperture_gm.c b/drivers/gpu/drm/i915/gvt/aperture_gm.c index 380eeb2a0e83..fe754022e356 100644 --- a/drivers/gpu/drm/i915/gvt/aperture_gm.c +++ b/drivers/gpu/drm/i915/gvt/aperture_gm.c @@ -131,7 +131,7 @@ void intel_vgpu_write_fence(struct intel_vgpu *vgpu, assert_rpm_wakelock_held(dev_priv); - if (WARN_ON(fence > vgpu_fence_sz(vgpu))) + if (WARN_ON(fence >= vgpu_fence_sz(vgpu))) return; reg = vgpu->fence.regs[fence]; diff --git a/drivers/gpu/drm/i915/gvt/cmd_parser.c b/drivers/gpu/drm/i915/gvt/cmd_parser.c index 45e89b1e0481..a614db310ea2 100644 --- a/drivers/gpu/drm/i915/gvt/cmd_parser.c +++ b/drivers/gpu/drm/i915/gvt/cmd_parser.c @@ -874,7 +874,7 @@ static int cmd_reg_handler(struct parser_exec_state *s, if (!intel_gvt_mmio_is_cmd_access(gvt, offset)) { gvt_vgpu_err("%s access to non-render register (%x)\n", cmd, offset); - return 0; + return -EBADRQC; } if (is_shadowed_mmio(offset)) { diff --git a/drivers/gpu/drm/i915/gvt/gvt.c b/drivers/gpu/drm/i915/gvt/gvt.c index 712f9d14e720..46c8b720e336 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.c +++ b/drivers/gpu/drm/i915/gvt/gvt.c @@ -176,6 +176,7 @@ static const struct intel_gvt_ops intel_gvt_ops = { .emulate_mmio_write = intel_vgpu_emulate_mmio_write, .vgpu_create = intel_gvt_create_vgpu, .vgpu_destroy = intel_gvt_destroy_vgpu, + .vgpu_release = intel_gvt_release_vgpu, .vgpu_reset = intel_gvt_reset_vgpu, .vgpu_activate = intel_gvt_activate_vgpu, .vgpu_deactivate = intel_gvt_deactivate_vgpu, @@ -315,6 +316,11 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv) if (WARN_ON(!gvt)) return; + intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); + intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt); + intel_gvt_cleanup_vgpu_type_groups(gvt); + intel_gvt_clean_vgpu_types(gvt); + intel_gvt_debugfs_clean(gvt); clean_service_thread(gvt); intel_gvt_clean_cmd_parser(gvt); @@ -322,17 +328,10 @@ void intel_gvt_clean_device(struct drm_i915_private *dev_priv) intel_gvt_clean_workload_scheduler(gvt); intel_gvt_clean_gtt(gvt); intel_gvt_clean_irq(gvt); - intel_gvt_clean_mmio_info(gvt); intel_gvt_free_firmware(gvt); - - intel_gvt_hypervisor_host_exit(&dev_priv->drm.pdev->dev, gvt); - intel_gvt_cleanup_vgpu_type_groups(gvt); - intel_gvt_clean_vgpu_types(gvt); - + intel_gvt_clean_mmio_info(gvt); idr_destroy(&gvt->vgpu_idr); - intel_gvt_destroy_idle_vgpu(gvt->idle_vgpu); - kfree(dev_priv->gvt); dev_priv->gvt = NULL; } diff --git a/drivers/gpu/drm/i915/gvt/gvt.h b/drivers/gpu/drm/i915/gvt/gvt.h index 9a9671522774..31f6cdbe5c42 100644 --- a/drivers/gpu/drm/i915/gvt/gvt.h +++ b/drivers/gpu/drm/i915/gvt/gvt.h @@ -486,6 +486,7 @@ void intel_gvt_destroy_idle_vgpu(struct intel_vgpu *vgpu); struct intel_vgpu *intel_gvt_create_vgpu(struct intel_gvt *gvt, struct intel_vgpu_type *type); void intel_gvt_destroy_vgpu(struct intel_vgpu *vgpu); +void intel_gvt_release_vgpu(struct intel_vgpu *vgpu); void intel_gvt_reset_vgpu_locked(struct intel_vgpu *vgpu, bool dmlr, unsigned int engine_mask); void intel_gvt_reset_vgpu(struct intel_vgpu *vgpu); @@ -563,7 +564,8 @@ struct intel_gvt_ops { unsigned int); struct intel_vgpu *(*vgpu_create)(struct intel_gvt *, struct intel_vgpu_type *); - void (*vgpu_destroy)(struct intel_vgpu *); + void (*vgpu_destroy)(struct intel_vgpu *vgpu); + void (*vgpu_release)(struct intel_vgpu *vgpu); void (*vgpu_reset)(struct intel_vgpu *); void (*vgpu_activate)(struct intel_vgpu *); void (*vgpu_deactivate)(struct intel_vgpu *); diff --git a/drivers/gpu/drm/i915/gvt/kvmgt.c b/drivers/gpu/drm/i915/gvt/kvmgt.c index 4d2f53ae9f0f..a45f46d8537f 100644 --- a/drivers/gpu/drm/i915/gvt/kvmgt.c +++ b/drivers/gpu/drm/i915/gvt/kvmgt.c @@ -43,6 +43,8 @@ #include <linux/mdev.h> #include <linux/debugfs.h> +#include <linux/nospec.h> + #include "i915_drv.h" #include "gvt.h" @@ -187,14 +189,14 @@ static int gvt_dma_map_page(struct intel_vgpu *vgpu, unsigned long gfn, /* Setup DMA mapping. */ *dma_addr = dma_map_page(dev, page, 0, size, PCI_DMA_BIDIRECTIONAL); - ret = dma_mapping_error(dev, *dma_addr); - if (ret) { + if (dma_mapping_error(dev, *dma_addr)) { gvt_vgpu_err("DMA mapping failed for pfn 0x%lx, ret %d\n", page_to_pfn(page), ret); gvt_unpin_guest_page(vgpu, gfn, size); + return -ENOMEM; } - return ret; + return 0; } static void gvt_dma_unmap_page(struct intel_vgpu *vgpu, unsigned long gfn, @@ -666,7 +668,7 @@ static void __intel_vgpu_release(struct intel_vgpu *vgpu) if (atomic_cmpxchg(&vgpu->vdev.released, 0, 1)) return; - intel_gvt_ops->vgpu_deactivate(vgpu); + intel_gvt_ops->vgpu_release(vgpu); ret = vfio_unregister_notifier(mdev_dev(vgpu->vdev.mdev), VFIO_IOMMU_NOTIFY, &vgpu->vdev.iommu_notifier); @@ -1139,7 +1141,8 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, } else if (cmd == VFIO_DEVICE_GET_REGION_INFO) { struct vfio_region_info info; struct vfio_info_cap caps = { .buf = NULL, .size = 0 }; - int i, ret; + unsigned int i; + int ret; struct vfio_region_info_cap_sparse_mmap *sparse = NULL; size_t size; int nr_areas = 1; @@ -1224,6 +1227,10 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, if (info.index >= VFIO_PCI_NUM_REGIONS + vgpu->vdev.num_regions) return -EINVAL; + info.index = + array_index_nospec(info.index, + VFIO_PCI_NUM_REGIONS + + vgpu->vdev.num_regions); i = info.index - VFIO_PCI_NUM_REGIONS; @@ -1250,11 +1257,13 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, &sparse->header, sizeof(*sparse) + (sparse->nr_areas * sizeof(*sparse->areas))); - kfree(sparse); - if (ret) + if (ret) { + kfree(sparse); return ret; + } break; default: + kfree(sparse); return -EINVAL; } } @@ -1270,6 +1279,7 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, sizeof(info), caps.buf, caps.size)) { kfree(caps.buf); + kfree(sparse); return -EFAULT; } info.cap_offset = sizeof(info); @@ -1278,6 +1288,7 @@ static long intel_vgpu_ioctl(struct mdev_device *mdev, unsigned int cmd, kfree(caps.buf); } + kfree(sparse); return copy_to_user((void __user *)arg, &info, minsz) ? -EFAULT : 0; } else if (cmd == VFIO_DEVICE_GET_IRQ_INFO) { @@ -1615,7 +1626,6 @@ static int kvmgt_guest_init(struct mdev_device *mdev) kvmgt_protect_table_init(info); gvt_cache_init(vgpu); - mutex_init(&vgpu->dmabuf_lock); init_completion(&vgpu->vblank_done); info->track_node.track_write = kvmgt_page_track_write; diff --git a/drivers/gpu/drm/i915/gvt/scheduler.c b/drivers/gpu/drm/i915/gvt/scheduler.c index b0e566956b8d..43aa058e29fc 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.c +++ b/drivers/gpu/drm/i915/gvt/scheduler.c @@ -784,7 +784,8 @@ static void update_guest_context(struct intel_vgpu_workload *workload) kunmap(page); } -static void clean_workloads(struct intel_vgpu *vgpu, unsigned long engine_mask) +void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu, + unsigned long engine_mask) { struct intel_vgpu_submission *s = &vgpu->submission; struct drm_i915_private *dev_priv = vgpu->gvt->dev_priv; @@ -879,7 +880,7 @@ static void complete_current_workload(struct intel_gvt *gvt, int ring_id) * cleaned up during the resetting process later, so doing * the workload clean up here doesn't have any impact. **/ - clean_workloads(vgpu, ENGINE_MASK(ring_id)); + intel_vgpu_clean_workloads(vgpu, ENGINE_MASK(ring_id)); } workload->complete(workload); @@ -1081,7 +1082,7 @@ void intel_vgpu_reset_submission(struct intel_vgpu *vgpu, if (!s->active) return; - clean_workloads(vgpu, engine_mask); + intel_vgpu_clean_workloads(vgpu, engine_mask); s->ops->reset(vgpu, engine_mask); } diff --git a/drivers/gpu/drm/i915/gvt/scheduler.h b/drivers/gpu/drm/i915/gvt/scheduler.h index 21eddab4a9cd..ca5529d0e48e 100644 --- a/drivers/gpu/drm/i915/gvt/scheduler.h +++ b/drivers/gpu/drm/i915/gvt/scheduler.h @@ -158,4 +158,7 @@ intel_vgpu_create_workload(struct intel_vgpu *vgpu, int ring_id, void intel_vgpu_destroy_workload(struct intel_vgpu_workload *workload); +void intel_vgpu_clean_workloads(struct intel_vgpu *vgpu, + unsigned long engine_mask); + #endif diff --git a/drivers/gpu/drm/i915/gvt/vgpu.c b/drivers/gpu/drm/i915/gvt/vgpu.c index f6fa916517c3..a4e8e3cf74fd 100644 --- a/drivers/gpu/drm/i915/gvt/vgpu.c +++ b/drivers/gpu/drm/i915/gvt/vgpu.c @@ -222,7 +222,7 @@ void intel_gvt_activate_vgpu(struct intel_vgpu *vgpu) * @vgpu: virtual GPU * * This function is called when user wants to deactivate a virtual GPU. - * All virtual GPU runtime information will be destroyed. + * The virtual GPU will be stopped. * */ void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu) @@ -238,12 +238,30 @@ void intel_gvt_deactivate_vgpu(struct intel_vgpu *vgpu) } intel_vgpu_stop_schedule(vgpu); - intel_vgpu_dmabuf_cleanup(vgpu); mutex_unlock(&vgpu->vgpu_lock); } /** + * intel_gvt_release_vgpu - release a virtual GPU + * @vgpu: virtual GPU + * + * This function is called when user wants to release a virtual GPU. + * The virtual GPU will be stopped and all runtime information will be + * destroyed. + * + */ +void intel_gvt_release_vgpu(struct intel_vgpu *vgpu) +{ + intel_gvt_deactivate_vgpu(vgpu); + + mutex_lock(&vgpu->vgpu_lock); + intel_vgpu_clean_workloads(vgpu, ALL_ENGINES); + intel_vgpu_dmabuf_cleanup(vgpu); + mutex_unlock(&vgpu->vgpu_lock); +} + +/** * intel_gvt_destroy_vgpu - destroy a virtual GPU * @vgpu: virtual GPU * @@ -361,6 +379,7 @@ static struct intel_vgpu *__intel_gvt_create_vgpu(struct intel_gvt *gvt, vgpu->gvt = gvt; vgpu->sched_ctl.weight = param->weight; mutex_init(&vgpu->vgpu_lock); + mutex_init(&vgpu->dmabuf_lock); INIT_LIST_HEAD(&vgpu->dmabuf_obj_list_head); INIT_RADIX_TREE(&vgpu->page_track_tree, GFP_KERNEL); idr_init(&vgpu->object_idr); diff --git a/drivers/gpu/drm/i915/i915_reg.h b/drivers/gpu/drm/i915/i915_reg.h index 91e7483228e1..08ec7446282e 100644 --- a/drivers/gpu/drm/i915/i915_reg.h +++ b/drivers/gpu/drm/i915/i915_reg.h @@ -9201,6 +9201,7 @@ enum skl_power_gate { #define TRANS_MSA_10_BPC (2 << 5) #define TRANS_MSA_12_BPC (3 << 5) #define TRANS_MSA_16_BPC (4 << 5) +#define TRANS_MSA_CEA_RANGE (1 << 3) /* LCPLL Control */ #define LCPLL_CTL _MMIO(0x130040) diff --git a/drivers/gpu/drm/i915/intel_ddi.c b/drivers/gpu/drm/i915/intel_ddi.c index 39d66f8493fa..8761513f3532 100644 --- a/drivers/gpu/drm/i915/intel_ddi.c +++ b/drivers/gpu/drm/i915/intel_ddi.c @@ -1685,6 +1685,10 @@ void intel_ddi_set_pipe_settings(const struct intel_crtc_state *crtc_state) WARN_ON(transcoder_is_dsi(cpu_transcoder)); temp = TRANS_MSA_SYNC_CLK; + + if (crtc_state->limited_color_range) + temp |= TRANS_MSA_CEA_RANGE; + switch (crtc_state->pipe_bpp) { case 18: temp |= TRANS_MSA_6_BPC; diff --git a/drivers/gpu/drm/i915/intel_ringbuffer.c b/drivers/gpu/drm/i915/intel_ringbuffer.c index 33faad3197fe..6a8f27d0a742 100644 --- a/drivers/gpu/drm/i915/intel_ringbuffer.c +++ b/drivers/gpu/drm/i915/intel_ringbuffer.c @@ -387,8 +387,18 @@ static void intel_ring_setup_status_page(struct intel_engine_cs *engine) mmio = RING_HWS_PGA(engine->mmio_base); } - if (INTEL_GEN(dev_priv) >= 6) - I915_WRITE(RING_HWSTAM(engine->mmio_base), 0xffffffff); + if (INTEL_GEN(dev_priv) >= 6) { + u32 mask = ~0u; + + /* + * Keep the render interrupt unmasked as this papers over + * lost interrupts following a reset. + */ + if (engine->id == RCS) + mask &= ~BIT(0); + + I915_WRITE(RING_HWSTAM(engine->mmio_base), mask); + } I915_WRITE(mmio, engine->status_page.ggtt_offset); POSTING_READ(mmio); diff --git a/drivers/gpu/drm/i915/intel_uncore.c b/drivers/gpu/drm/i915/intel_uncore.c index b892ca8396e8..50b39aa4ffb8 100644 --- a/drivers/gpu/drm/i915/intel_uncore.c +++ b/drivers/gpu/drm/i915/intel_uncore.c @@ -359,8 +359,8 @@ intel_uncore_fw_release_timer(struct hrtimer *timer) } /* Note callers must have acquired the PUNIT->PMIC bus, before calling this. */ -static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv, - bool restore) +static unsigned int +intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv) { unsigned long irqflags; struct intel_uncore_forcewake_domain *domain; @@ -412,20 +412,11 @@ static void intel_uncore_forcewake_reset(struct drm_i915_private *dev_priv, dev_priv->uncore.funcs.force_wake_put(dev_priv, fw); fw_domains_reset(dev_priv, dev_priv->uncore.fw_domains); - - if (restore) { /* If reset with a user forcewake, try to restore */ - if (fw) - dev_priv->uncore.funcs.force_wake_get(dev_priv, fw); - - if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) - dev_priv->uncore.fifo_count = - fifo_free_entries(dev_priv); - } - - if (!restore) - assert_forcewakes_inactive(dev_priv); + assert_forcewakes_inactive(dev_priv); spin_unlock_irqrestore(&dev_priv->uncore.lock, irqflags); + + return fw; /* track the lost user forcewake domains */ } static u64 gen9_edram_size(struct drm_i915_private *dev_priv) @@ -534,7 +525,7 @@ check_for_unclaimed_mmio(struct drm_i915_private *dev_priv) } static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, - bool restore_forcewake) + unsigned int restore_forcewake) { /* clear out unclaimed reg detection bit */ if (check_for_unclaimed_mmio(dev_priv)) @@ -549,7 +540,17 @@ static void __intel_uncore_early_sanitize(struct drm_i915_private *dev_priv, } iosf_mbi_punit_acquire(); - intel_uncore_forcewake_reset(dev_priv, restore_forcewake); + intel_uncore_forcewake_reset(dev_priv); + if (restore_forcewake) { + spin_lock_irq(&dev_priv->uncore.lock); + dev_priv->uncore.funcs.force_wake_get(dev_priv, + restore_forcewake); + + if (IS_GEN6(dev_priv) || IS_GEN7(dev_priv)) + dev_priv->uncore.fifo_count = + fifo_free_entries(dev_priv); + spin_unlock_irq(&dev_priv->uncore.lock); + } iosf_mbi_punit_release(); } @@ -558,13 +559,18 @@ void intel_uncore_suspend(struct drm_i915_private *dev_priv) iosf_mbi_punit_acquire(); iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( &dev_priv->uncore.pmic_bus_access_nb); - intel_uncore_forcewake_reset(dev_priv, false); + dev_priv->uncore.fw_domains_saved = + intel_uncore_forcewake_reset(dev_priv); iosf_mbi_punit_release(); } void intel_uncore_resume_early(struct drm_i915_private *dev_priv) { - __intel_uncore_early_sanitize(dev_priv, true); + unsigned int restore_forcewake; + + restore_forcewake = fetch_and_zero(&dev_priv->uncore.fw_domains_saved); + __intel_uncore_early_sanitize(dev_priv, restore_forcewake); + iosf_mbi_register_pmic_bus_access_notifier( &dev_priv->uncore.pmic_bus_access_nb); i915_check_and_clear_faults(dev_priv); @@ -1545,7 +1551,7 @@ void intel_uncore_init(struct drm_i915_private *dev_priv) intel_uncore_edram_detect(dev_priv); intel_uncore_fw_domains_init(dev_priv); - __intel_uncore_early_sanitize(dev_priv, false); + __intel_uncore_early_sanitize(dev_priv, 0); dev_priv->uncore.unclaimed_mmio_check = 1; dev_priv->uncore.pmic_bus_access_nb.notifier_call = @@ -1632,7 +1638,7 @@ void intel_uncore_fini(struct drm_i915_private *dev_priv) iosf_mbi_punit_acquire(); iosf_mbi_unregister_pmic_bus_access_notifier_unlocked( &dev_priv->uncore.pmic_bus_access_nb); - intel_uncore_forcewake_reset(dev_priv, false); + intel_uncore_forcewake_reset(dev_priv); iosf_mbi_punit_release(); } diff --git a/drivers/gpu/drm/i915/intel_uncore.h b/drivers/gpu/drm/i915/intel_uncore.h index 2fbe93178fb2..e5e157d288de 100644 --- a/drivers/gpu/drm/i915/intel_uncore.h +++ b/drivers/gpu/drm/i915/intel_uncore.h @@ -104,6 +104,7 @@ struct intel_uncore { enum forcewake_domains fw_domains; enum forcewake_domains fw_domains_active; + enum forcewake_domains fw_domains_saved; /* user domains saved for S3 */ u32 fw_set; u32 fw_clear; diff --git a/drivers/gpu/drm/i915/selftests/i915_gem_object.c b/drivers/gpu/drm/i915/selftests/i915_gem_object.c index c69cbd5aed52..ba4f322d56b8 100644 --- a/drivers/gpu/drm/i915/selftests/i915_gem_object.c +++ b/drivers/gpu/drm/i915/selftests/i915_gem_object.c @@ -499,6 +499,19 @@ static bool assert_mmap_offset(struct drm_i915_private *i915, return err == expected; } +static void disable_retire_worker(struct drm_i915_private *i915) +{ + mutex_lock(&i915->drm.struct_mutex); + if (!i915->gt.active_requests++) { + intel_runtime_pm_get(i915); + i915_gem_unpark(i915); + intel_runtime_pm_put(i915); + } + mutex_unlock(&i915->drm.struct_mutex); + cancel_delayed_work_sync(&i915->gt.retire_work); + cancel_delayed_work_sync(&i915->gt.idle_work); +} + static int igt_mmap_offset_exhaustion(void *arg) { struct drm_i915_private *i915 = arg; @@ -509,12 +522,7 @@ static int igt_mmap_offset_exhaustion(void *arg) int loop, err; /* Disable background reaper */ - mutex_lock(&i915->drm.struct_mutex); - if (!i915->gt.active_requests++) - i915_gem_unpark(i915); - mutex_unlock(&i915->drm.struct_mutex); - cancel_delayed_work_sync(&i915->gt.retire_work); - cancel_delayed_work_sync(&i915->gt.idle_work); + disable_retire_worker(i915); GEM_BUG_ON(!i915->gt.awake); /* Trim the device mmap space to only a page */ diff --git a/drivers/gpu/drm/i915/selftests/intel_uncore.c b/drivers/gpu/drm/i915/selftests/intel_uncore.c index 47bc5b2ddb56..81d9d31042a9 100644 --- a/drivers/gpu/drm/i915/selftests/intel_uncore.c +++ b/drivers/gpu/drm/i915/selftests/intel_uncore.c @@ -160,7 +160,7 @@ static int intel_uncore_check_forcewake_domains(struct drm_i915_private *dev_pri i915_reg_t reg = { offset }; iosf_mbi_punit_acquire(); - intel_uncore_forcewake_reset(dev_priv, false); + intel_uncore_forcewake_reset(dev_priv); iosf_mbi_punit_release(); check_for_unclaimed_mmio(dev_priv); diff --git a/drivers/gpu/drm/msm/Makefile b/drivers/gpu/drm/msm/Makefile index 7c773e003663..261fa79d456d 100644 --- a/drivers/gpu/drm/msm/Makefile +++ b/drivers/gpu/drm/msm/Makefile @@ -11,6 +11,9 @@ msm-y := \ adreno/a5xx_gpu.o \ adreno/a5xx_power.o \ adreno/a5xx_preempt.o \ + adreno/a6xx_gpu.o \ + adreno/a6xx_gmu.o \ + adreno/a6xx_hfi.o \ hdmi/hdmi.o \ hdmi/hdmi_audio.o \ hdmi/hdmi_bridge.o \ diff --git a/drivers/gpu/drm/msm/adreno/a2xx.xml.h b/drivers/gpu/drm/msm/adreno/a2xx.xml.h index 644374c7b3e0..4bff0a740c7d 100644 --- a/drivers/gpu/drm/msm/adreno/a2xx.xml.h +++ b/drivers/gpu/drm/msm/adreno/a2xx.xml.h @@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 431 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 37162 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 13324 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 31866 bytes, from 2017-06-06 18:26:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 111898 bytes, from 2017-06-06 18:23:59) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml ( 139480 bytes, from 2017-06-16 12:44:39) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/adreno.xml ( 501 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml ( 36805 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml ( 13634 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml ( 42393 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml ( 112086 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml ( 147240 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml ( 101627 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml ( 10431 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) @@ -84,13 +86,12 @@ enum a2xx_sq_surfaceformat { FMT_5_5_5_1 = 13, FMT_8_8_8_8_A = 14, FMT_4_4_4_4 = 15, - FMT_10_11_11 = 16, - FMT_11_11_10 = 17, + FMT_8_8_8 = 16, FMT_DXT1 = 18, FMT_DXT2_3 = 19, FMT_DXT4_5 = 20, + FMT_10_10_10_2 = 21, FMT_24_8 = 22, - FMT_24_8_FLOAT = 23, FMT_16 = 24, FMT_16_16 = 25, FMT_16_16_16_16 = 26, @@ -106,29 +107,23 @@ enum a2xx_sq_surfaceformat { FMT_32_FLOAT = 36, FMT_32_32_FLOAT = 37, FMT_32_32_32_32_FLOAT = 38, - FMT_32_AS_8 = 39, - FMT_32_AS_8_8 = 40, - FMT_16_MPEG = 41, - FMT_16_16_MPEG = 42, - FMT_8_INTERLACED = 43, - FMT_32_AS_8_INTERLACED = 44, - FMT_32_AS_8_8_INTERLACED = 45, - FMT_16_INTERLACED = 46, - FMT_16_MPEG_INTERLACED = 47, - FMT_16_16_MPEG_INTERLACED = 48, + FMT_ATI_TC_RGB = 39, + FMT_ATI_TC_RGBA = 40, + FMT_ATI_TC_555_565_RGB = 41, + FMT_ATI_TC_555_565_RGBA = 42, + FMT_ATI_TC_RGBA_INTERP = 43, + FMT_ATI_TC_555_565_RGBA_INTERP = 44, + FMT_ETC1_RGBA_INTERP = 46, + FMT_ETC1_RGB = 47, + FMT_ETC1_RGBA = 48, FMT_DXN = 49, - FMT_8_8_8_8_AS_16_16_16_16 = 50, - FMT_DXT1_AS_16_16_16_16 = 51, - FMT_DXT2_3_AS_16_16_16_16 = 52, - FMT_DXT4_5_AS_16_16_16_16 = 53, + FMT_2_3_3 = 51, FMT_2_10_10_10_AS_16_16_16_16 = 54, - FMT_10_11_11_AS_16_16_16_16 = 55, - FMT_11_11_10_AS_16_16_16_16 = 56, + FMT_10_10_10_2_AS_16_16_16_16 = 55, FMT_32_32_32_FLOAT = 57, FMT_DXT3A = 58, FMT_DXT5A = 59, FMT_CTX1 = 60, - FMT_DXT3A_AS_1_1_1_1 = 61, }; enum a2xx_sq_ps_vtx_mode { diff --git a/drivers/gpu/drm/msm/adreno/a3xx.xml.h b/drivers/gpu/drm/msm/adreno/a3xx.xml.h index 663a73216926..645a19aef399 100644 --- a/drivers/gpu/drm/msm/adreno/a3xx.xml.h +++ b/drivers/gpu/drm/msm/adreno/a3xx.xml.h @@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 431 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 37162 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 13324 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 31866 bytes, from 2017-06-06 18:26:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 111898 bytes, from 2017-06-06 18:23:59) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml ( 139480 bytes, from 2017-06-16 12:44:39) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/adreno.xml ( 501 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml ( 36805 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml ( 13634 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml ( 42393 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml ( 112086 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml ( 147240 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml ( 101627 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml ( 10431 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) diff --git a/drivers/gpu/drm/msm/adreno/a4xx.xml.h b/drivers/gpu/drm/msm/adreno/a4xx.xml.h index 1a14f4a40b9c..19565e87aa7b 100644 --- a/drivers/gpu/drm/msm/adreno/a4xx.xml.h +++ b/drivers/gpu/drm/msm/adreno/a4xx.xml.h @@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 431 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 37162 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 13324 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 31866 bytes, from 2017-06-06 18:26:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 111898 bytes, from 2017-06-06 18:23:59) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml ( 139480 bytes, from 2017-06-16 12:44:39) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/adreno.xml ( 501 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml ( 36805 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml ( 13634 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml ( 42393 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml ( 112086 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml ( 147240 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml ( 101627 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml ( 10431 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) @@ -263,12 +265,6 @@ enum a4xx_depth_format { DEPTH4_32 = 3, }; -enum a4xx_tess_spacing { - EQUAL_SPACING = 0, - ODD_SPACING = 2, - EVEN_SPACING = 3, -}; - enum a4xx_ccu_perfcounter_select { CCU_BUSY_CYCLES = 0, CCU_RB_DEPTH_RETURN_STALL = 2, @@ -3544,12 +3540,13 @@ static inline uint32_t A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH(uint32_t val) { return ((val) << A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_VS_CONTROL_REG_CONSTLENGTH__MASK; } -#define A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x0000ff00 +#define A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x00007f00 #define A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT 8 static inline uint32_t A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val) { return ((val) << A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_VS_CONTROL_REG_CONSTOBJECTOFFSET__MASK; } +#define A4XX_HLSQ_VS_CONTROL_REG_SSBO_ENABLE 0x00008000 #define A4XX_HLSQ_VS_CONTROL_REG_ENABLED 0x00010000 #define A4XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__MASK 0x00fe0000 #define A4XX_HLSQ_VS_CONTROL_REG_SHADEROBJOFFSET__SHIFT 17 @@ -3571,12 +3568,13 @@ static inline uint32_t A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH(uint32_t val) { return ((val) << A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_FS_CONTROL_REG_CONSTLENGTH__MASK; } -#define A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x0000ff00 +#define A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x00007f00 #define A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT 8 static inline uint32_t A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val) { return ((val) << A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_FS_CONTROL_REG_CONSTOBJECTOFFSET__MASK; } +#define A4XX_HLSQ_FS_CONTROL_REG_SSBO_ENABLE 0x00008000 #define A4XX_HLSQ_FS_CONTROL_REG_ENABLED 0x00010000 #define A4XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__MASK 0x00fe0000 #define A4XX_HLSQ_FS_CONTROL_REG_SHADEROBJOFFSET__SHIFT 17 @@ -3598,12 +3596,13 @@ static inline uint32_t A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH(uint32_t val) { return ((val) << A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_HS_CONTROL_REG_CONSTLENGTH__MASK; } -#define A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x0000ff00 +#define A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x00007f00 #define A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT 8 static inline uint32_t A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val) { return ((val) << A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_HS_CONTROL_REG_CONSTOBJECTOFFSET__MASK; } +#define A4XX_HLSQ_HS_CONTROL_REG_SSBO_ENABLE 0x00008000 #define A4XX_HLSQ_HS_CONTROL_REG_ENABLED 0x00010000 #define A4XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__MASK 0x00fe0000 #define A4XX_HLSQ_HS_CONTROL_REG_SHADEROBJOFFSET__SHIFT 17 @@ -3625,12 +3624,13 @@ static inline uint32_t A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH(uint32_t val) { return ((val) << A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_DS_CONTROL_REG_CONSTLENGTH__MASK; } -#define A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x0000ff00 +#define A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x00007f00 #define A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT 8 static inline uint32_t A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val) { return ((val) << A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_DS_CONTROL_REG_CONSTOBJECTOFFSET__MASK; } +#define A4XX_HLSQ_DS_CONTROL_REG_SSBO_ENABLE 0x00008000 #define A4XX_HLSQ_DS_CONTROL_REG_ENABLED 0x00010000 #define A4XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__MASK 0x00fe0000 #define A4XX_HLSQ_DS_CONTROL_REG_SHADEROBJOFFSET__SHIFT 17 @@ -3652,12 +3652,13 @@ static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH(uint32_t val) { return ((val) << A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_CONSTLENGTH__MASK; } -#define A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x0000ff00 +#define A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x00007f00 #define A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT 8 static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val) { return ((val) << A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_CONSTOBJECTOFFSET__MASK; } +#define A4XX_HLSQ_GS_CONTROL_REG_SSBO_ENABLE 0x00008000 #define A4XX_HLSQ_GS_CONTROL_REG_ENABLED 0x00010000 #define A4XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__MASK 0x00fe0000 #define A4XX_HLSQ_GS_CONTROL_REG_SHADEROBJOFFSET__SHIFT 17 @@ -3672,23 +3673,103 @@ static inline uint32_t A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH(uint32_t val) return ((val) << A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_GS_CONTROL_REG_INSTRLENGTH__MASK; } -#define REG_A4XX_HLSQ_CS_CONTROL 0x000023ca +#define REG_A4XX_HLSQ_CS_CONTROL_REG 0x000023ca +#define A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__MASK 0x000000ff +#define A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__SHIFT 0 +static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_CONSTLENGTH__MASK; +} +#define A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__MASK 0x00007f00 +#define A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT 8 +static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_CONSTOBJECTOFFSET__MASK; +} +#define A4XX_HLSQ_CS_CONTROL_REG_SSBO_ENABLE 0x00008000 +#define A4XX_HLSQ_CS_CONTROL_REG_ENABLED 0x00010000 +#define A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__MASK 0x00fe0000 +#define A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__SHIFT 17 +static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_SHADEROBJOFFSET__MASK; +} +#define A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__MASK 0xff000000 +#define A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__SHIFT 24 +static inline uint32_t A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__SHIFT) & A4XX_HLSQ_CS_CONTROL_REG_INSTRLENGTH__MASK; +} #define REG_A4XX_HLSQ_CL_NDRANGE_0 0x000023cd +#define A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__MASK 0x00000003 +#define A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__SHIFT 0 +static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_KERNELDIM__MASK; +} +#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__MASK 0x00000ffc +#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__SHIFT 2 +static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEX__MASK; +} +#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__MASK 0x003ff000 +#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__SHIFT 12 +static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEY__MASK; +} +#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__MASK 0xffc00000 +#define A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__SHIFT 22 +static inline uint32_t A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__SHIFT) & A4XX_HLSQ_CL_NDRANGE_0_LOCALSIZEZ__MASK; +} #define REG_A4XX_HLSQ_CL_NDRANGE_1 0x000023ce +#define A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__MASK 0xffffffff +#define A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__SHIFT 0 +static inline uint32_t A4XX_HLSQ_CL_NDRANGE_1_SIZE_X(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__SHIFT) & A4XX_HLSQ_CL_NDRANGE_1_SIZE_X__MASK; +} #define REG_A4XX_HLSQ_CL_NDRANGE_2 0x000023cf #define REG_A4XX_HLSQ_CL_NDRANGE_3 0x000023d0 +#define A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__MASK 0xffffffff +#define A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__SHIFT 0 +static inline uint32_t A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__SHIFT) & A4XX_HLSQ_CL_NDRANGE_3_SIZE_Y__MASK; +} #define REG_A4XX_HLSQ_CL_NDRANGE_4 0x000023d1 #define REG_A4XX_HLSQ_CL_NDRANGE_5 0x000023d2 +#define A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__MASK 0xffffffff +#define A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__SHIFT 0 +static inline uint32_t A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__SHIFT) & A4XX_HLSQ_CL_NDRANGE_5_SIZE_Z__MASK; +} #define REG_A4XX_HLSQ_CL_NDRANGE_6 0x000023d3 #define REG_A4XX_HLSQ_CL_CONTROL_0 0x000023d4 +#define A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__MASK 0x000000ff +#define A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__SHIFT 0 +static inline uint32_t A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__SHIFT) & A4XX_HLSQ_CL_CONTROL_0_WGIDCONSTID__MASK; +} +#define A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__MASK 0xff000000 +#define A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__SHIFT 24 +static inline uint32_t A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID(uint32_t val) +{ + return ((val) << A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__SHIFT) & A4XX_HLSQ_CL_CONTROL_0_LOCALIDREGID__MASK; +} #define REG_A4XX_HLSQ_CL_CONTROL_1 0x000023d5 @@ -4087,5 +4168,71 @@ static inline uint32_t A4XX_TEX_CONST_4_BASE(uint32_t val) #define REG_A4XX_TEX_CONST_7 0x00000007 +#define REG_A4XX_SSBO_0_0 0x00000000 +#define A4XX_SSBO_0_0_BASE__MASK 0xffffffe0 +#define A4XX_SSBO_0_0_BASE__SHIFT 5 +static inline uint32_t A4XX_SSBO_0_0_BASE(uint32_t val) +{ + return ((val >> 5) << A4XX_SSBO_0_0_BASE__SHIFT) & A4XX_SSBO_0_0_BASE__MASK; +} + +#define REG_A4XX_SSBO_0_1 0x00000001 +#define A4XX_SSBO_0_1_PITCH__MASK 0x003fffff +#define A4XX_SSBO_0_1_PITCH__SHIFT 0 +static inline uint32_t A4XX_SSBO_0_1_PITCH(uint32_t val) +{ + return ((val) << A4XX_SSBO_0_1_PITCH__SHIFT) & A4XX_SSBO_0_1_PITCH__MASK; +} + +#define REG_A4XX_SSBO_0_2 0x00000002 +#define A4XX_SSBO_0_2_ARRAY_PITCH__MASK 0x03fff000 +#define A4XX_SSBO_0_2_ARRAY_PITCH__SHIFT 12 +static inline uint32_t A4XX_SSBO_0_2_ARRAY_PITCH(uint32_t val) +{ + return ((val >> 12) << A4XX_SSBO_0_2_ARRAY_PITCH__SHIFT) & A4XX_SSBO_0_2_ARRAY_PITCH__MASK; +} + +#define REG_A4XX_SSBO_0_3 0x00000003 +#define A4XX_SSBO_0_3_CPP__MASK 0x0000003f +#define A4XX_SSBO_0_3_CPP__SHIFT 0 +static inline uint32_t A4XX_SSBO_0_3_CPP(uint32_t val) +{ + return ((val) << A4XX_SSBO_0_3_CPP__SHIFT) & A4XX_SSBO_0_3_CPP__MASK; +} + +#define REG_A4XX_SSBO_1_0 0x00000000 +#define A4XX_SSBO_1_0_CPP__MASK 0x0000001f +#define A4XX_SSBO_1_0_CPP__SHIFT 0 +static inline uint32_t A4XX_SSBO_1_0_CPP(uint32_t val) +{ + return ((val) << A4XX_SSBO_1_0_CPP__SHIFT) & A4XX_SSBO_1_0_CPP__MASK; +} +#define A4XX_SSBO_1_0_FMT__MASK 0x0000ff00 +#define A4XX_SSBO_1_0_FMT__SHIFT 8 +static inline uint32_t A4XX_SSBO_1_0_FMT(enum a4xx_color_fmt val) +{ + return ((val) << A4XX_SSBO_1_0_FMT__SHIFT) & A4XX_SSBO_1_0_FMT__MASK; +} +#define A4XX_SSBO_1_0_WIDTH__MASK 0xffff0000 +#define A4XX_SSBO_1_0_WIDTH__SHIFT 16 +static inline uint32_t A4XX_SSBO_1_0_WIDTH(uint32_t val) +{ + return ((val) << A4XX_SSBO_1_0_WIDTH__SHIFT) & A4XX_SSBO_1_0_WIDTH__MASK; +} + +#define REG_A4XX_SSBO_1_1 0x00000001 +#define A4XX_SSBO_1_1_HEIGHT__MASK 0x0000ffff +#define A4XX_SSBO_1_1_HEIGHT__SHIFT 0 +static inline uint32_t A4XX_SSBO_1_1_HEIGHT(uint32_t val) +{ + return ((val) << A4XX_SSBO_1_1_HEIGHT__SHIFT) & A4XX_SSBO_1_1_HEIGHT__MASK; +} +#define A4XX_SSBO_1_1_DEPTH__MASK 0xffff0000 +#define A4XX_SSBO_1_1_DEPTH__SHIFT 16 +static inline uint32_t A4XX_SSBO_1_1_DEPTH(uint32_t val) +{ + return ((val) << A4XX_SSBO_1_1_DEPTH__SHIFT) & A4XX_SSBO_1_1_DEPTH__MASK; +} + #endif /* A4XX_XML */ diff --git a/drivers/gpu/drm/msm/adreno/a5xx.xml.h b/drivers/gpu/drm/msm/adreno/a5xx.xml.h index e0e6711f4f78..182d37ff3794 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx.xml.h +++ b/drivers/gpu/drm/msm/adreno/a5xx.xml.h @@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 431 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 37162 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 13324 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 31866 bytes, from 2017-06-06 18:26:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 111898 bytes, from 2017-06-06 18:23:59) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml ( 139480 bytes, from 2017-06-16 12:44:39) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/adreno.xml ( 501 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml ( 36805 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml ( 13634 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml ( 42393 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml ( 112086 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml ( 147240 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml ( 101627 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml ( 10431 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) @@ -119,6 +121,11 @@ enum a5xx_vtx_fmt { VFMT5_8_8_8_8_SNORM = 50, VFMT5_8_8_8_8_UINT = 51, VFMT5_8_8_8_8_SINT = 52, + VFMT5_10_10_10_2_UNORM = 54, + VFMT5_10_10_10_2_SNORM = 57, + VFMT5_10_10_10_2_UINT = 58, + VFMT5_10_10_10_2_SINT = 59, + VFMT5_11_11_10_FLOAT = 66, VFMT5_16_16_UNORM = 67, VFMT5_16_16_SNORM = 68, VFMT5_16_16_FLOAT = 69, @@ -204,14 +211,45 @@ enum a5xx_tex_fmt { TFMT5_32_32_FLOAT = 103, TFMT5_32_32_UINT = 104, TFMT5_32_32_SINT = 105, + TFMT5_32_32_32_UINT = 114, + TFMT5_32_32_32_SINT = 115, + TFMT5_32_32_32_FLOAT = 116, TFMT5_32_32_32_32_FLOAT = 130, TFMT5_32_32_32_32_UINT = 131, TFMT5_32_32_32_32_SINT = 132, TFMT5_X8Z24_UNORM = 160, + TFMT5_ETC2_RG11_UNORM = 171, + TFMT5_ETC2_RG11_SNORM = 172, + TFMT5_ETC2_R11_UNORM = 173, + TFMT5_ETC2_R11_SNORM = 174, + TFMT5_ETC1 = 175, + TFMT5_ETC2_RGB8 = 176, + TFMT5_ETC2_RGBA8 = 177, + TFMT5_ETC2_RGB8A1 = 178, + TFMT5_DXT1 = 179, + TFMT5_DXT3 = 180, + TFMT5_DXT5 = 181, TFMT5_RGTC1_UNORM = 183, TFMT5_RGTC1_SNORM = 184, TFMT5_RGTC2_UNORM = 187, TFMT5_RGTC2_SNORM = 188, + TFMT5_BPTC_UFLOAT = 190, + TFMT5_BPTC_FLOAT = 191, + TFMT5_BPTC = 192, + TFMT5_ASTC_4x4 = 193, + TFMT5_ASTC_5x4 = 194, + TFMT5_ASTC_5x5 = 195, + TFMT5_ASTC_6x5 = 196, + TFMT5_ASTC_6x6 = 197, + TFMT5_ASTC_8x5 = 198, + TFMT5_ASTC_8x6 = 199, + TFMT5_ASTC_8x8 = 200, + TFMT5_ASTC_10x5 = 201, + TFMT5_ASTC_10x6 = 202, + TFMT5_ASTC_10x8 = 203, + TFMT5_ASTC_10x10 = 204, + TFMT5_ASTC_12x10 = 205, + TFMT5_ASTC_12x12 = 206, }; enum a5xx_tex_fetchsize { @@ -239,7 +277,7 @@ enum a5xx_blit_buf { BLIT_MRT6 = 6, BLIT_MRT7 = 7, BLIT_ZS = 8, - BLIT_Z32 = 9, + BLIT_S = 9, }; enum a5xx_cp_perfcounter_select { @@ -899,6 +937,12 @@ enum a5xx_tex_type { #define REG_A5XX_CP_DRAW_STATE_DATA 0x0000080c +#define REG_A5XX_CP_ME_NRT_ADDR_LO 0x0000080d + +#define REG_A5XX_CP_ME_NRT_ADDR_HI 0x0000080e + +#define REG_A5XX_CP_ME_NRT_DATA 0x00000810 + #define REG_A5XX_CP_CRASH_SCRIPT_BASE_LO 0x00000817 #define REG_A5XX_CP_CRASH_SCRIPT_BASE_HI 0x00000818 @@ -2072,9 +2116,17 @@ static inline uint32_t A5XX_VSC_RESOLVE_CNTL_Y(uint32_t val) #define REG_A5XX_PC_MODE_CNTL 0x00000d02 -#define REG_A5XX_UNKNOWN_0D08 0x00000d08 +#define REG_A5XX_PC_INDEX_BUF_LO 0x00000d04 + +#define REG_A5XX_PC_INDEX_BUF_HI 0x00000d05 + +#define REG_A5XX_PC_START_INDEX 0x00000d06 -#define REG_A5XX_UNKNOWN_0D09 0x00000d09 +#define REG_A5XX_PC_MAX_INDEX 0x00000d07 + +#define REG_A5XX_PC_TESSFACTOR_ADDR_LO 0x00000d08 + +#define REG_A5XX_PC_TESSFACTOR_ADDR_HI 0x00000d09 #define REG_A5XX_PC_PERFCTR_PC_SEL_0 0x00000d10 @@ -2327,6 +2379,14 @@ static inline uint32_t A5XX_VSC_RESOLVE_CNTL_Y(uint32_t val) #define REG_A5XX_VBIF_PERF_CNT_EN3 0x000030c3 +#define REG_A5XX_VBIF_PERF_CNT_CLR0 0x000030c8 + +#define REG_A5XX_VBIF_PERF_CNT_CLR1 0x000030c9 + +#define REG_A5XX_VBIF_PERF_CNT_CLR2 0x000030ca + +#define REG_A5XX_VBIF_PERF_CNT_CLR3 0x000030cb + #define REG_A5XX_VBIF_PERF_CNT_SEL0 0x000030d0 #define REG_A5XX_VBIF_PERF_CNT_SEL1 0x000030d1 @@ -2590,6 +2650,7 @@ static inline uint32_t A5XX_VSC_RESOLVE_CNTL_Y(uint32_t val) #define REG_A5XX_GPU_CS_AMP_CALIBRATION_CONTROL1 0x0000c557 #define REG_A5XX_GRAS_CL_CNTL 0x0000e000 +#define A5XX_GRAS_CL_CNTL_ZERO_GB_SCALE_Z 0x00000040 #define REG_A5XX_UNKNOWN_E001 0x0000e001 @@ -2700,7 +2761,7 @@ static inline uint32_t A5XX_GRAS_SU_POINT_SIZE(float val) return ((((int32_t)(val * 16.0))) << A5XX_GRAS_SU_POINT_SIZE__SHIFT) & A5XX_GRAS_SU_POINT_SIZE__MASK; } -#define REG_A5XX_UNKNOWN_E093 0x0000e093 +#define REG_A5XX_GRAS_SU_LAYERED 0x0000e093 #define REG_A5XX_GRAS_SU_DEPTH_PLANE_CNTL 0x0000e094 #define A5XX_GRAS_SU_DEPTH_PLANE_CNTL_FRAG_WRITES_Z 0x00000001 @@ -2936,7 +2997,9 @@ static inline uint32_t A5XX_RB_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val #define A5XX_RB_RENDER_CONTROL0_WCOORD 0x00000200 #define REG_A5XX_RB_RENDER_CONTROL1 0x0000e145 +#define A5XX_RB_RENDER_CONTROL1_SAMPLEMASK 0x00000001 #define A5XX_RB_RENDER_CONTROL1_FACENESS 0x00000002 +#define A5XX_RB_RENDER_CONTROL1_SAMPLEID 0x00000004 #define REG_A5XX_RB_FS_OUTPUT_CNTL 0x0000e146 #define A5XX_RB_FS_OUTPUT_CNTL_MRT__MASK 0x0000000f @@ -3002,6 +3065,13 @@ static inline uint32_t REG_A5XX_RB_MRT(uint32_t i0) { return 0x0000e150 + 0x7*i0 static inline uint32_t REG_A5XX_RB_MRT_CONTROL(uint32_t i0) { return 0x0000e150 + 0x7*i0; } #define A5XX_RB_MRT_CONTROL_BLEND 0x00000001 #define A5XX_RB_MRT_CONTROL_BLEND2 0x00000002 +#define A5XX_RB_MRT_CONTROL_ROP_ENABLE 0x00000004 +#define A5XX_RB_MRT_CONTROL_ROP_CODE__MASK 0x00000078 +#define A5XX_RB_MRT_CONTROL_ROP_CODE__SHIFT 3 +static inline uint32_t A5XX_RB_MRT_CONTROL_ROP_CODE(enum a3xx_rop_code val) +{ + return ((val) << A5XX_RB_MRT_CONTROL_ROP_CODE__SHIFT) & A5XX_RB_MRT_CONTROL_ROP_CODE__MASK; +} #define A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK 0x00000780 #define A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT 7 static inline uint32_t A5XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val) @@ -3060,6 +3130,12 @@ static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a5xx_tile_mode { return ((val) << A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A5XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK; } +#define A5XX_RB_MRT_BUF_INFO_DITHER_MODE__MASK 0x00001800 +#define A5XX_RB_MRT_BUF_INFO_DITHER_MODE__SHIFT 11 +static inline uint32_t A5XX_RB_MRT_BUF_INFO_DITHER_MODE(enum adreno_rb_dither_mode val) +{ + return ((val) << A5XX_RB_MRT_BUF_INFO_DITHER_MODE__SHIFT) & A5XX_RB_MRT_BUF_INFO_DITHER_MODE__MASK; +} #define A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK 0x00006000 #define A5XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT 13 static inline uint32_t A5XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val) @@ -3223,6 +3299,7 @@ static inline uint32_t A5XX_RB_BLEND_CNTL_ENABLE_BLEND(uint32_t val) return ((val) << A5XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT) & A5XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK; } #define A5XX_RB_BLEND_CNTL_INDEPENDENT_BLEND 0x00000100 +#define A5XX_RB_BLEND_CNTL_ALPHA_TO_COVERAGE 0x00000400 #define A5XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK 0xffff0000 #define A5XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT 16 static inline uint32_t A5XX_RB_BLEND_CNTL_SAMPLE_MASK(uint32_t val) @@ -3369,7 +3446,25 @@ static inline uint32_t A5XX_RB_STENCILREFMASK_STENCILWRITEMASK(uint32_t val) return ((val) << A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__SHIFT) & A5XX_RB_STENCILREFMASK_STENCILWRITEMASK__MASK; } -#define REG_A5XX_UNKNOWN_E1C7 0x0000e1c7 +#define REG_A5XX_RB_STENCILREFMASK_BF 0x0000e1c7 +#define A5XX_RB_STENCILREFMASK_BF_STENCILREF__MASK 0x000000ff +#define A5XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT 0 +static inline uint32_t A5XX_RB_STENCILREFMASK_BF_STENCILREF(uint32_t val) +{ + return ((val) << A5XX_RB_STENCILREFMASK_BF_STENCILREF__SHIFT) & A5XX_RB_STENCILREFMASK_BF_STENCILREF__MASK; +} +#define A5XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK 0x0000ff00 +#define A5XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT 8 +static inline uint32_t A5XX_RB_STENCILREFMASK_BF_STENCILMASK(uint32_t val) +{ + return ((val) << A5XX_RB_STENCILREFMASK_BF_STENCILMASK__SHIFT) & A5XX_RB_STENCILREFMASK_BF_STENCILMASK__MASK; +} +#define A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK 0x00ff0000 +#define A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT 16 +static inline uint32_t A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK(uint32_t val) +{ + return ((val) << A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__SHIFT) & A5XX_RB_STENCILREFMASK_BF_STENCILWRITEMASK__MASK; +} #define REG_A5XX_RB_WINDOW_OFFSET 0x0000e1d0 #define A5XX_RB_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE 0x80000000 @@ -3428,6 +3523,7 @@ static inline uint32_t A5XX_RB_RESOLVE_CNTL_2_Y(uint32_t val) } #define REG_A5XX_RB_RESOLVE_CNTL_3 0x0000e213 +#define A5XX_RB_RESOLVE_CNTL_3_TILED 0x00000001 #define REG_A5XX_RB_BLIT_DST_LO 0x0000e214 @@ -3459,6 +3555,7 @@ static inline uint32_t A5XX_RB_BLIT_DST_ARRAY_PITCH(uint32_t val) #define REG_A5XX_RB_CLEAR_CNTL 0x0000e21c #define A5XX_RB_CLEAR_CNTL_FAST_CLEAR 0x00000002 +#define A5XX_RB_CLEAR_CNTL_MSAA_RESOLVE 0x00000004 #define A5XX_RB_CLEAR_CNTL_MASK__MASK 0x000000f0 #define A5XX_RB_CLEAR_CNTL_MASK__SHIFT 4 static inline uint32_t A5XX_RB_CLEAR_CNTL_MASK(uint32_t val) @@ -3627,22 +3724,69 @@ static inline uint32_t A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC(uint32_t val) { return ((val) << A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__SHIFT) & A5XX_PC_PRIMITIVE_CNTL_STRIDE_IN_VPC__MASK; } +#define A5XX_PC_PRIMITIVE_CNTL_PRIMITIVE_RESTART 0x00000100 +#define A5XX_PC_PRIMITIVE_CNTL_COUNT_PRIMITIVES 0x00000200 #define A5XX_PC_PRIMITIVE_CNTL_PROVOKING_VTX_LAST 0x00000400 #define REG_A5XX_PC_PRIM_VTX_CNTL 0x0000e385 #define A5XX_PC_PRIM_VTX_CNTL_PSIZE 0x00000800 #define REG_A5XX_PC_RASTER_CNTL 0x0000e388 +#define A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__MASK 0x00000007 +#define A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__SHIFT 0 +static inline uint32_t A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE(enum adreno_pa_su_sc_draw val) +{ + return ((val) << A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__SHIFT) & A5XX_PC_RASTER_CNTL_POLYMODE_FRONT_PTYPE__MASK; +} +#define A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__MASK 0x00000038 +#define A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__SHIFT 3 +static inline uint32_t A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE(enum adreno_pa_su_sc_draw val) +{ + return ((val) << A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__SHIFT) & A5XX_PC_RASTER_CNTL_POLYMODE_BACK_PTYPE__MASK; +} +#define A5XX_PC_RASTER_CNTL_POLYMODE_ENABLE 0x00000040 #define REG_A5XX_UNKNOWN_E389 0x0000e389 #define REG_A5XX_PC_RESTART_INDEX 0x0000e38c -#define REG_A5XX_UNKNOWN_E38D 0x0000e38d +#define REG_A5XX_PC_GS_LAYERED 0x0000e38d #define REG_A5XX_PC_GS_PARAM 0x0000e38e +#define A5XX_PC_GS_PARAM_MAX_VERTICES__MASK 0x000003ff +#define A5XX_PC_GS_PARAM_MAX_VERTICES__SHIFT 0 +static inline uint32_t A5XX_PC_GS_PARAM_MAX_VERTICES(uint32_t val) +{ + return ((val) << A5XX_PC_GS_PARAM_MAX_VERTICES__SHIFT) & A5XX_PC_GS_PARAM_MAX_VERTICES__MASK; +} +#define A5XX_PC_GS_PARAM_INVOCATIONS__MASK 0x0000f800 +#define A5XX_PC_GS_PARAM_INVOCATIONS__SHIFT 11 +static inline uint32_t A5XX_PC_GS_PARAM_INVOCATIONS(uint32_t val) +{ + return ((val) << A5XX_PC_GS_PARAM_INVOCATIONS__SHIFT) & A5XX_PC_GS_PARAM_INVOCATIONS__MASK; +} +#define A5XX_PC_GS_PARAM_PRIMTYPE__MASK 0x01800000 +#define A5XX_PC_GS_PARAM_PRIMTYPE__SHIFT 23 +static inline uint32_t A5XX_PC_GS_PARAM_PRIMTYPE(enum adreno_pa_su_sc_draw val) +{ + return ((val) << A5XX_PC_GS_PARAM_PRIMTYPE__SHIFT) & A5XX_PC_GS_PARAM_PRIMTYPE__MASK; +} #define REG_A5XX_PC_HS_PARAM 0x0000e38f +#define A5XX_PC_HS_PARAM_VERTICES_OUT__MASK 0x0000003f +#define A5XX_PC_HS_PARAM_VERTICES_OUT__SHIFT 0 +static inline uint32_t A5XX_PC_HS_PARAM_VERTICES_OUT(uint32_t val) +{ + return ((val) << A5XX_PC_HS_PARAM_VERTICES_OUT__SHIFT) & A5XX_PC_HS_PARAM_VERTICES_OUT__MASK; +} +#define A5XX_PC_HS_PARAM_SPACING__MASK 0x00600000 +#define A5XX_PC_HS_PARAM_SPACING__SHIFT 21 +static inline uint32_t A5XX_PC_HS_PARAM_SPACING(enum a4xx_tess_spacing val) +{ + return ((val) << A5XX_PC_HS_PARAM_SPACING__SHIFT) & A5XX_PC_HS_PARAM_SPACING__MASK; +} +#define A5XX_PC_HS_PARAM_CW 0x00800000 +#define A5XX_PC_HS_PARAM_CONNECTED 0x01000000 #define REG_A5XX_PC_POWER_CNTL 0x0000e3b0 @@ -3667,10 +3811,40 @@ static inline uint32_t A5XX_VFD_CONTROL_1_REGID4INST(uint32_t val) { return ((val) << A5XX_VFD_CONTROL_1_REGID4INST__SHIFT) & A5XX_VFD_CONTROL_1_REGID4INST__MASK; } +#define A5XX_VFD_CONTROL_1_REGID4PRIMID__MASK 0x00ff0000 +#define A5XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT 16 +static inline uint32_t A5XX_VFD_CONTROL_1_REGID4PRIMID(uint32_t val) +{ + return ((val) << A5XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT) & A5XX_VFD_CONTROL_1_REGID4PRIMID__MASK; +} #define REG_A5XX_VFD_CONTROL_2 0x0000e402 +#define A5XX_VFD_CONTROL_2_REGID_PATCHID__MASK 0x000000ff +#define A5XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT 0 +static inline uint32_t A5XX_VFD_CONTROL_2_REGID_PATCHID(uint32_t val) +{ + return ((val) << A5XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT) & A5XX_VFD_CONTROL_2_REGID_PATCHID__MASK; +} #define REG_A5XX_VFD_CONTROL_3 0x0000e403 +#define A5XX_VFD_CONTROL_3_REGID_PATCHID__MASK 0x0000ff00 +#define A5XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT 8 +static inline uint32_t A5XX_VFD_CONTROL_3_REGID_PATCHID(uint32_t val) +{ + return ((val) << A5XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT) & A5XX_VFD_CONTROL_3_REGID_PATCHID__MASK; +} +#define A5XX_VFD_CONTROL_3_REGID_TESSX__MASK 0x00ff0000 +#define A5XX_VFD_CONTROL_3_REGID_TESSX__SHIFT 16 +static inline uint32_t A5XX_VFD_CONTROL_3_REGID_TESSX(uint32_t val) +{ + return ((val) << A5XX_VFD_CONTROL_3_REGID_TESSX__SHIFT) & A5XX_VFD_CONTROL_3_REGID_TESSX__MASK; +} +#define A5XX_VFD_CONTROL_3_REGID_TESSY__MASK 0xff000000 +#define A5XX_VFD_CONTROL_3_REGID_TESSY__SHIFT 24 +static inline uint32_t A5XX_VFD_CONTROL_3_REGID_TESSY(uint32_t val) +{ + return ((val) << A5XX_VFD_CONTROL_3_REGID_TESSY__SHIFT) & A5XX_VFD_CONTROL_3_REGID_TESSY__MASK; +} #define REG_A5XX_VFD_CONTROL_4 0x0000e404 @@ -3700,12 +3874,18 @@ static inline uint32_t A5XX_VFD_DECODE_INSTR_IDX(uint32_t val) return ((val) << A5XX_VFD_DECODE_INSTR_IDX__SHIFT) & A5XX_VFD_DECODE_INSTR_IDX__MASK; } #define A5XX_VFD_DECODE_INSTR_INSTANCED 0x00020000 -#define A5XX_VFD_DECODE_INSTR_FORMAT__MASK 0x3ff00000 +#define A5XX_VFD_DECODE_INSTR_FORMAT__MASK 0x0ff00000 #define A5XX_VFD_DECODE_INSTR_FORMAT__SHIFT 20 static inline uint32_t A5XX_VFD_DECODE_INSTR_FORMAT(enum a5xx_vtx_fmt val) { return ((val) << A5XX_VFD_DECODE_INSTR_FORMAT__SHIFT) & A5XX_VFD_DECODE_INSTR_FORMAT__MASK; } +#define A5XX_VFD_DECODE_INSTR_SWAP__MASK 0x30000000 +#define A5XX_VFD_DECODE_INSTR_SWAP__SHIFT 28 +static inline uint32_t A5XX_VFD_DECODE_INSTR_SWAP(enum a3xx_color_swap val) +{ + return ((val) << A5XX_VFD_DECODE_INSTR_SWAP__SHIFT) & A5XX_VFD_DECODE_INSTR_SWAP__MASK; +} #define A5XX_VFD_DECODE_INSTR_UNK30 0x40000000 #define A5XX_VFD_DECODE_INSTR_FLOAT 0x80000000 @@ -3960,6 +4140,7 @@ static inline uint32_t A5XX_SP_FS_CTRL_REG0_BRANCHSTACK(uint32_t val) #define REG_A5XX_SP_BLEND_CNTL 0x0000e5c9 #define A5XX_SP_BLEND_CNTL_ENABLED 0x00000001 #define A5XX_SP_BLEND_CNTL_UNK8 0x00000100 +#define A5XX_SP_BLEND_CNTL_ALPHA_TO_COVERAGE 0x00000400 #define REG_A5XX_SP_FS_OUTPUT_CNTL 0x0000e5ca #define A5XX_SP_FS_OUTPUT_CNTL_MRT__MASK 0x0000000f @@ -4001,16 +4182,12 @@ static inline uint32_t A5XX_SP_FS_MRT_REG_COLOR_FORMAT(enum a5xx_color_fmt val) { return ((val) << A5XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT) & A5XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK; } +#define A5XX_SP_FS_MRT_REG_COLOR_SINT 0x00000100 +#define A5XX_SP_FS_MRT_REG_COLOR_UINT 0x00000200 #define A5XX_SP_FS_MRT_REG_COLOR_SRGB 0x00000400 #define REG_A5XX_UNKNOWN_E5DB 0x0000e5db -#define REG_A5XX_UNKNOWN_E5F2 0x0000e5f2 - -#define REG_A5XX_SP_CS_OBJ_START_LO 0x0000e5f3 - -#define REG_A5XX_SP_CS_OBJ_START_HI 0x0000e5f4 - #define REG_A5XX_SP_CS_CTRL_REG0 0x0000e5f0 #define A5XX_SP_CS_CTRL_REG0_THREADSIZE__MASK 0x00000008 #define A5XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT 3 @@ -4039,7 +4216,39 @@ static inline uint32_t A5XX_SP_CS_CTRL_REG0_BRANCHSTACK(uint32_t val) return ((val) << A5XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK; } -#define REG_A5XX_UNKNOWN_E600 0x0000e600 +#define REG_A5XX_UNKNOWN_E5F2 0x0000e5f2 + +#define REG_A5XX_SP_CS_OBJ_START_LO 0x0000e5f3 + +#define REG_A5XX_SP_CS_OBJ_START_HI 0x0000e5f4 + +#define REG_A5XX_SP_HS_CTRL_REG0 0x0000e600 +#define A5XX_SP_HS_CTRL_REG0_THREADSIZE__MASK 0x00000008 +#define A5XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT 3 +static inline uint32_t A5XX_SP_HS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) +{ + return ((val) << A5XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_HS_CTRL_REG0_THREADSIZE__MASK; +} +#define A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x000003f0 +#define A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 4 +static inline uint32_t A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) +{ + return ((val) << A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK; +} +#define A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0000fc00 +#define A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 10 +static inline uint32_t A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) +{ + return ((val) << A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK; +} +#define A5XX_SP_HS_CTRL_REG0_VARYING 0x00010000 +#define A5XX_SP_HS_CTRL_REG0_PIXLODENABLE 0x00100000 +#define A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK 0xfe000000 +#define A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT 25 +static inline uint32_t A5XX_SP_HS_CTRL_REG0_BRANCHSTACK(uint32_t val) +{ + return ((val) << A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK; +} #define REG_A5XX_UNKNOWN_E602 0x0000e602 @@ -4047,13 +4256,67 @@ static inline uint32_t A5XX_SP_CS_CTRL_REG0_BRANCHSTACK(uint32_t val) #define REG_A5XX_SP_HS_OBJ_START_HI 0x0000e604 +#define REG_A5XX_SP_DS_CTRL_REG0 0x0000e610 +#define A5XX_SP_DS_CTRL_REG0_THREADSIZE__MASK 0x00000008 +#define A5XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT 3 +static inline uint32_t A5XX_SP_DS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) +{ + return ((val) << A5XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_DS_CTRL_REG0_THREADSIZE__MASK; +} +#define A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x000003f0 +#define A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 4 +static inline uint32_t A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) +{ + return ((val) << A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK; +} +#define A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0000fc00 +#define A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 10 +static inline uint32_t A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) +{ + return ((val) << A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK; +} +#define A5XX_SP_DS_CTRL_REG0_VARYING 0x00010000 +#define A5XX_SP_DS_CTRL_REG0_PIXLODENABLE 0x00100000 +#define A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK 0xfe000000 +#define A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT 25 +static inline uint32_t A5XX_SP_DS_CTRL_REG0_BRANCHSTACK(uint32_t val) +{ + return ((val) << A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK; +} + #define REG_A5XX_UNKNOWN_E62B 0x0000e62b #define REG_A5XX_SP_DS_OBJ_START_LO 0x0000e62c #define REG_A5XX_SP_DS_OBJ_START_HI 0x0000e62d -#define REG_A5XX_UNKNOWN_E640 0x0000e640 +#define REG_A5XX_SP_GS_CTRL_REG0 0x0000e640 +#define A5XX_SP_GS_CTRL_REG0_THREADSIZE__MASK 0x00000008 +#define A5XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT 3 +static inline uint32_t A5XX_SP_GS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) +{ + return ((val) << A5XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT) & A5XX_SP_GS_CTRL_REG0_THREADSIZE__MASK; +} +#define A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x000003f0 +#define A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 4 +static inline uint32_t A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) +{ + return ((val) << A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A5XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK; +} +#define A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x0000fc00 +#define A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 10 +static inline uint32_t A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) +{ + return ((val) << A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A5XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK; +} +#define A5XX_SP_GS_CTRL_REG0_VARYING 0x00010000 +#define A5XX_SP_GS_CTRL_REG0_PIXLODENABLE 0x00100000 +#define A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK 0xfe000000 +#define A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT 25 +static inline uint32_t A5XX_SP_GS_CTRL_REG0_BRANCHSTACK(uint32_t val) +{ + return ((val) << A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT) & A5XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK; +} #define REG_A5XX_UNKNOWN_E65B 0x0000e65b @@ -4173,6 +4436,18 @@ static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val) { return ((val) << A5XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK; } +#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK 0x0000ff00 +#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT 8 +static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_SAMPLEID(uint32_t val) +{ + return ((val) << A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK; +} +#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK 0x00ff0000 +#define A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT 16 +static inline uint32_t A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val) +{ + return ((val) << A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A5XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK; +} #define REG_A5XX_HLSQ_CONTROL_3_REG 0x0000e787 #define A5XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__MASK 0x000000ff @@ -4375,34 +4650,52 @@ static inline uint32_t A5XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ(uint32_t val) } #define REG_A5XX_HLSQ_CS_NDRANGE_1 0x0000e7b1 -#define A5XX_HLSQ_CS_NDRANGE_1_SIZE_X__MASK 0xffffffff -#define A5XX_HLSQ_CS_NDRANGE_1_SIZE_X__SHIFT 0 -static inline uint32_t A5XX_HLSQ_CS_NDRANGE_1_SIZE_X(uint32_t val) +#define A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK 0xffffffff +#define A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT 0 +static inline uint32_t A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X(uint32_t val) { - return ((val) << A5XX_HLSQ_CS_NDRANGE_1_SIZE_X__SHIFT) & A5XX_HLSQ_CS_NDRANGE_1_SIZE_X__MASK; + return ((val) << A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT) & A5XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK; } #define REG_A5XX_HLSQ_CS_NDRANGE_2 0x0000e7b2 +#define A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK 0xffffffff +#define A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT 0 +static inline uint32_t A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X(uint32_t val) +{ + return ((val) << A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT) & A5XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK; +} #define REG_A5XX_HLSQ_CS_NDRANGE_3 0x0000e7b3 -#define A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y__MASK 0xffffffff -#define A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y__SHIFT 0 -static inline uint32_t A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y(uint32_t val) +#define A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK 0xffffffff +#define A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT 0 +static inline uint32_t A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y(uint32_t val) { - return ((val) << A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y__SHIFT) & A5XX_HLSQ_CS_NDRANGE_3_SIZE_Y__MASK; + return ((val) << A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT) & A5XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK; } #define REG_A5XX_HLSQ_CS_NDRANGE_4 0x0000e7b4 +#define A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK 0xffffffff +#define A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT 0 +static inline uint32_t A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y(uint32_t val) +{ + return ((val) << A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT) & A5XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK; +} #define REG_A5XX_HLSQ_CS_NDRANGE_5 0x0000e7b5 -#define A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z__MASK 0xffffffff -#define A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z__SHIFT 0 -static inline uint32_t A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z(uint32_t val) +#define A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK 0xffffffff +#define A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT 0 +static inline uint32_t A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z(uint32_t val) { - return ((val) << A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z__SHIFT) & A5XX_HLSQ_CS_NDRANGE_5_SIZE_Z__MASK; + return ((val) << A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT) & A5XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK; } #define REG_A5XX_HLSQ_CS_NDRANGE_6 0x0000e7b6 +#define A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK 0xffffffff +#define A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT 0 +static inline uint32_t A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z(uint32_t val) +{ + return ((val) << A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT) & A5XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK; +} #define REG_A5XX_HLSQ_CS_CNTL_0 0x0000e7b7 #define A5XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK 0x000000ff @@ -4468,6 +4761,8 @@ static inline uint32_t A5XX_HLSQ_CS_CNTL_0_LOCALIDREGID(uint32_t val) #define REG_A5XX_HLSQ_CS_INSTRLEN 0x0000e7dd +#define REG_A5XX_RB_2D_BLIT_CNTL 0x00002100 + #define REG_A5XX_RB_2D_SRC_SOLID_DW0 0x00002101 #define REG_A5XX_RB_2D_SRC_SOLID_DW1 0x00002102 @@ -4483,12 +4778,19 @@ static inline uint32_t A5XX_RB_2D_SRC_INFO_COLOR_FORMAT(enum a5xx_color_fmt val) { return ((val) << A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_2D_SRC_INFO_COLOR_FORMAT__MASK; } +#define A5XX_RB_2D_SRC_INFO_TILE_MODE__MASK 0x00000300 +#define A5XX_RB_2D_SRC_INFO_TILE_MODE__SHIFT 8 +static inline uint32_t A5XX_RB_2D_SRC_INFO_TILE_MODE(enum a5xx_tile_mode val) +{ + return ((val) << A5XX_RB_2D_SRC_INFO_TILE_MODE__SHIFT) & A5XX_RB_2D_SRC_INFO_TILE_MODE__MASK; +} #define A5XX_RB_2D_SRC_INFO_COLOR_SWAP__MASK 0x00000c00 #define A5XX_RB_2D_SRC_INFO_COLOR_SWAP__SHIFT 10 static inline uint32_t A5XX_RB_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val) { return ((val) << A5XX_RB_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_2D_SRC_INFO_COLOR_SWAP__MASK; } +#define A5XX_RB_2D_SRC_INFO_FLAGS 0x00001000 #define REG_A5XX_RB_2D_SRC_LO 0x00002108 @@ -4515,12 +4817,19 @@ static inline uint32_t A5XX_RB_2D_DST_INFO_COLOR_FORMAT(enum a5xx_color_fmt val) { return ((val) << A5XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A5XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK; } +#define A5XX_RB_2D_DST_INFO_TILE_MODE__MASK 0x00000300 +#define A5XX_RB_2D_DST_INFO_TILE_MODE__SHIFT 8 +static inline uint32_t A5XX_RB_2D_DST_INFO_TILE_MODE(enum a5xx_tile_mode val) +{ + return ((val) << A5XX_RB_2D_DST_INFO_TILE_MODE__SHIFT) & A5XX_RB_2D_DST_INFO_TILE_MODE__MASK; +} #define A5XX_RB_2D_DST_INFO_COLOR_SWAP__MASK 0x00000c00 #define A5XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT 10 static inline uint32_t A5XX_RB_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val) { return ((val) << A5XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT) & A5XX_RB_2D_DST_INFO_COLOR_SWAP__MASK; } +#define A5XX_RB_2D_DST_INFO_FLAGS 0x00001000 #define REG_A5XX_RB_2D_DST_LO 0x00002111 @@ -4548,6 +4857,8 @@ static inline uint32_t A5XX_RB_2D_DST_SIZE_ARRAY_PITCH(uint32_t val) #define REG_A5XX_RB_2D_DST_FLAGS_HI 0x00002144 +#define REG_A5XX_GRAS_2D_BLIT_CNTL 0x00002180 + #define REG_A5XX_GRAS_2D_SRC_INFO 0x00002181 #define A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__MASK 0x000000ff #define A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__SHIFT 0 @@ -4555,12 +4866,19 @@ static inline uint32_t A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT(enum a5xx_color_fmt va { return ((val) << A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A5XX_GRAS_2D_SRC_INFO_COLOR_FORMAT__MASK; } +#define A5XX_GRAS_2D_SRC_INFO_TILE_MODE__MASK 0x00000300 +#define A5XX_GRAS_2D_SRC_INFO_TILE_MODE__SHIFT 8 +static inline uint32_t A5XX_GRAS_2D_SRC_INFO_TILE_MODE(enum a5xx_tile_mode val) +{ + return ((val) << A5XX_GRAS_2D_SRC_INFO_TILE_MODE__SHIFT) & A5XX_GRAS_2D_SRC_INFO_TILE_MODE__MASK; +} #define A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__MASK 0x00000c00 #define A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__SHIFT 10 static inline uint32_t A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val) { return ((val) << A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A5XX_GRAS_2D_SRC_INFO_COLOR_SWAP__MASK; } +#define A5XX_GRAS_2D_SRC_INFO_FLAGS 0x00001000 #define REG_A5XX_GRAS_2D_DST_INFO 0x00002182 #define A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__MASK 0x000000ff @@ -4569,12 +4887,19 @@ static inline uint32_t A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT(enum a5xx_color_fmt va { return ((val) << A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A5XX_GRAS_2D_DST_INFO_COLOR_FORMAT__MASK; } +#define A5XX_GRAS_2D_DST_INFO_TILE_MODE__MASK 0x00000300 +#define A5XX_GRAS_2D_DST_INFO_TILE_MODE__SHIFT 8 +static inline uint32_t A5XX_GRAS_2D_DST_INFO_TILE_MODE(enum a5xx_tile_mode val) +{ + return ((val) << A5XX_GRAS_2D_DST_INFO_TILE_MODE__SHIFT) & A5XX_GRAS_2D_DST_INFO_TILE_MODE__MASK; +} #define A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__MASK 0x00000c00 #define A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__SHIFT 10 static inline uint32_t A5XX_GRAS_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val) { return ((val) << A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__SHIFT) & A5XX_GRAS_2D_DST_INFO_COLOR_SWAP__MASK; } +#define A5XX_GRAS_2D_DST_INFO_FLAGS 0x00001000 #define REG_A5XX_UNKNOWN_2100 0x00002100 @@ -4698,6 +5023,12 @@ static inline uint32_t A5XX_TEX_CONST_0_MIPLVLS(uint32_t val) { return ((val) << A5XX_TEX_CONST_0_MIPLVLS__SHIFT) & A5XX_TEX_CONST_0_MIPLVLS__MASK; } +#define A5XX_TEX_CONST_0_SAMPLES__MASK 0x00300000 +#define A5XX_TEX_CONST_0_SAMPLES__SHIFT 20 +static inline uint32_t A5XX_TEX_CONST_0_SAMPLES(enum a3xx_msaa_samples val) +{ + return ((val) << A5XX_TEX_CONST_0_SAMPLES__SHIFT) & A5XX_TEX_CONST_0_SAMPLES__MASK; +} #define A5XX_TEX_CONST_0_FMT__MASK 0x3fc00000 #define A5XX_TEX_CONST_0_FMT__SHIFT 22 static inline uint32_t A5XX_TEX_CONST_0_FMT(enum a5xx_tex_fmt val) @@ -4788,5 +5119,81 @@ static inline uint32_t A5XX_TEX_CONST_5_DEPTH(uint32_t val) #define REG_A5XX_TEX_CONST_11 0x0000000b +#define REG_A5XX_SSBO_0_0 0x00000000 +#define A5XX_SSBO_0_0_BASE_LO__MASK 0xffffffe0 +#define A5XX_SSBO_0_0_BASE_LO__SHIFT 5 +static inline uint32_t A5XX_SSBO_0_0_BASE_LO(uint32_t val) +{ + return ((val >> 5) << A5XX_SSBO_0_0_BASE_LO__SHIFT) & A5XX_SSBO_0_0_BASE_LO__MASK; +} + +#define REG_A5XX_SSBO_0_1 0x00000001 +#define A5XX_SSBO_0_1_PITCH__MASK 0x003fffff +#define A5XX_SSBO_0_1_PITCH__SHIFT 0 +static inline uint32_t A5XX_SSBO_0_1_PITCH(uint32_t val) +{ + return ((val) << A5XX_SSBO_0_1_PITCH__SHIFT) & A5XX_SSBO_0_1_PITCH__MASK; +} + +#define REG_A5XX_SSBO_0_2 0x00000002 +#define A5XX_SSBO_0_2_ARRAY_PITCH__MASK 0x03fff000 +#define A5XX_SSBO_0_2_ARRAY_PITCH__SHIFT 12 +static inline uint32_t A5XX_SSBO_0_2_ARRAY_PITCH(uint32_t val) +{ + return ((val >> 12) << A5XX_SSBO_0_2_ARRAY_PITCH__SHIFT) & A5XX_SSBO_0_2_ARRAY_PITCH__MASK; +} + +#define REG_A5XX_SSBO_0_3 0x00000003 +#define A5XX_SSBO_0_3_CPP__MASK 0x0000003f +#define A5XX_SSBO_0_3_CPP__SHIFT 0 +static inline uint32_t A5XX_SSBO_0_3_CPP(uint32_t val) +{ + return ((val) << A5XX_SSBO_0_3_CPP__SHIFT) & A5XX_SSBO_0_3_CPP__MASK; +} + +#define REG_A5XX_SSBO_1_0 0x00000000 +#define A5XX_SSBO_1_0_FMT__MASK 0x0000ff00 +#define A5XX_SSBO_1_0_FMT__SHIFT 8 +static inline uint32_t A5XX_SSBO_1_0_FMT(enum a5xx_tex_fmt val) +{ + return ((val) << A5XX_SSBO_1_0_FMT__SHIFT) & A5XX_SSBO_1_0_FMT__MASK; +} +#define A5XX_SSBO_1_0_WIDTH__MASK 0xffff0000 +#define A5XX_SSBO_1_0_WIDTH__SHIFT 16 +static inline uint32_t A5XX_SSBO_1_0_WIDTH(uint32_t val) +{ + return ((val) << A5XX_SSBO_1_0_WIDTH__SHIFT) & A5XX_SSBO_1_0_WIDTH__MASK; +} + +#define REG_A5XX_SSBO_1_1 0x00000001 +#define A5XX_SSBO_1_1_HEIGHT__MASK 0x0000ffff +#define A5XX_SSBO_1_1_HEIGHT__SHIFT 0 +static inline uint32_t A5XX_SSBO_1_1_HEIGHT(uint32_t val) +{ + return ((val) << A5XX_SSBO_1_1_HEIGHT__SHIFT) & A5XX_SSBO_1_1_HEIGHT__MASK; +} +#define A5XX_SSBO_1_1_DEPTH__MASK 0xffff0000 +#define A5XX_SSBO_1_1_DEPTH__SHIFT 16 +static inline uint32_t A5XX_SSBO_1_1_DEPTH(uint32_t val) +{ + return ((val) << A5XX_SSBO_1_1_DEPTH__SHIFT) & A5XX_SSBO_1_1_DEPTH__MASK; +} + +#define REG_A5XX_SSBO_2_0 0x00000000 +#define A5XX_SSBO_2_0_BASE_LO__MASK 0xffffffff +#define A5XX_SSBO_2_0_BASE_LO__SHIFT 0 +static inline uint32_t A5XX_SSBO_2_0_BASE_LO(uint32_t val) +{ + return ((val) << A5XX_SSBO_2_0_BASE_LO__SHIFT) & A5XX_SSBO_2_0_BASE_LO__MASK; +} + +#define REG_A5XX_SSBO_2_1 0x00000001 +#define A5XX_SSBO_2_1_BASE_HI__MASK 0xffffffff +#define A5XX_SSBO_2_1_BASE_HI__SHIFT 0 +static inline uint32_t A5XX_SSBO_2_1_BASE_HI(uint32_t val) +{ + return ((val) << A5XX_SSBO_2_1_BASE_HI__SHIFT) & A5XX_SSBO_2_1_BASE_HI__MASK; +} + #endif /* A5XX_XML */ diff --git a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c index bd84f71d27d8..ab1d9308c311 100644 --- a/drivers/gpu/drm/msm/adreno/a5xx_gpu.c +++ b/drivers/gpu/drm/msm/adreno/a5xx_gpu.c @@ -11,6 +11,7 @@ * */ +#include <linux/kernel.h> #include <linux/types.h> #include <linux/cpumask.h> #include <linux/qcom_scm.h> @@ -20,6 +21,7 @@ #include <linux/pm_opp.h> #include <linux/nvmem-consumer.h> #include <linux/iopoll.h> +#include <linux/slab.h> #include "msm_gem.h" #include "msm_mmu.h" #include "a5xx_gpu.h" @@ -92,12 +94,13 @@ static int zap_shader_load_mdt(struct msm_gpu *gpu, const char *fwname) ret = qcom_mdt_load(dev, fw, fwname, GPU_PAS_ID, mem_region, mem_phys, mem_size, NULL); } else { - char newname[strlen("qcom/") + strlen(fwname) + 1]; + char *newname; - sprintf(newname, "qcom/%s", fwname); + newname = kasprintf(GFP_KERNEL, "qcom/%s", fwname); ret = qcom_mdt_load(dev, fw, newname, GPU_PAS_ID, mem_region, mem_phys, mem_size, NULL); + kfree(newname); } if (ret) goto out; diff --git a/drivers/gpu/drm/msm/adreno/a6xx.xml.h b/drivers/gpu/drm/msm/adreno/a6xx.xml.h new file mode 100644 index 000000000000..87eab51f7000 --- /dev/null +++ b/drivers/gpu/drm/msm/adreno/a6xx.xml.h @@ -0,0 +1,4562 @@ +#ifndef A6XX_XML +#define A6XX_XML + +/* Autogenerated file, DO NOT EDIT manually! + +This file was generated by the rules-ng-ng headergen tool in this git repository: +http://github.com/freedreno/envytools/ +git clone https://github.com/freedreno/envytools.git + +The rules-ng-ng source files this header was generated from are: +- /home/robclark/src/envytools/rnndb/adreno.xml ( 501 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml ( 36805 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml ( 13634 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml ( 42393 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml ( 112086 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml ( 147240 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml ( 101627 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml ( 10431 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: +- Rob Clark <robdclark@gmail.com> (robclark) +- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice (including the +next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +enum a6xx_color_fmt { + RB6_A8_UNORM = 2, + RB6_R8_UNORM = 3, + RB6_R8_SNORM = 4, + RB6_R8_UINT = 5, + RB6_R8_SINT = 6, + RB6_R4G4B4A4_UNORM = 8, + RB6_R5G5B5A1_UNORM = 10, + RB6_R5G6B5_UNORM = 14, + RB6_R8G8_UNORM = 15, + RB6_R8G8_SNORM = 16, + RB6_R8G8_UINT = 17, + RB6_R8G8_SINT = 18, + RB6_R16_UNORM = 21, + RB6_R16_SNORM = 22, + RB6_R16_FLOAT = 23, + RB6_R16_UINT = 24, + RB6_R16_SINT = 25, + RB6_R8G8B8A8_UNORM = 48, + RB6_R8G8B8_UNORM = 49, + RB6_R8G8B8A8_SNORM = 50, + RB6_R8G8B8A8_UINT = 51, + RB6_R8G8B8A8_SINT = 52, + RB6_R10G10B10A2_UNORM = 55, + RB6_R10G10B10A2_UINT = 58, + RB6_R11G11B10_FLOAT = 66, + RB6_R16G16_UNORM = 67, + RB6_R16G16_SNORM = 68, + RB6_R16G16_FLOAT = 69, + RB6_R16G16_UINT = 70, + RB6_R16G16_SINT = 71, + RB6_R32_FLOAT = 74, + RB6_R32_UINT = 75, + RB6_R32_SINT = 76, + RB6_R16G16B16A16_UNORM = 96, + RB6_R16G16B16A16_SNORM = 97, + RB6_R16G16B16A16_FLOAT = 98, + RB6_R16G16B16A16_UINT = 99, + RB6_R16G16B16A16_SINT = 100, + RB6_R32G32_FLOAT = 103, + RB6_R32G32_UINT = 104, + RB6_R32G32_SINT = 105, + RB6_R32G32B32A32_FLOAT = 130, + RB6_R32G32B32A32_UINT = 131, + RB6_R32G32B32A32_SINT = 132, + RB6_X8Z24_UNORM = 160, +}; + +enum a6xx_tile_mode { + TILE6_LINEAR = 0, + TILE6_2 = 2, + TILE6_3 = 3, +}; + +enum a6xx_vtx_fmt { + VFMT6_8_UNORM = 3, + VFMT6_8_SNORM = 4, + VFMT6_8_UINT = 5, + VFMT6_8_SINT = 6, + VFMT6_8_8_UNORM = 15, + VFMT6_8_8_SNORM = 16, + VFMT6_8_8_UINT = 17, + VFMT6_8_8_SINT = 18, + VFMT6_16_UNORM = 21, + VFMT6_16_SNORM = 22, + VFMT6_16_FLOAT = 23, + VFMT6_16_UINT = 24, + VFMT6_16_SINT = 25, + VFMT6_8_8_8_UNORM = 33, + VFMT6_8_8_8_SNORM = 34, + VFMT6_8_8_8_UINT = 35, + VFMT6_8_8_8_SINT = 36, + VFMT6_8_8_8_8_UNORM = 48, + VFMT6_8_8_8_8_SNORM = 50, + VFMT6_8_8_8_8_UINT = 51, + VFMT6_8_8_8_8_SINT = 52, + VFMT6_10_10_10_2_UNORM = 54, + VFMT6_10_10_10_2_SNORM = 57, + VFMT6_10_10_10_2_UINT = 58, + VFMT6_10_10_10_2_SINT = 59, + VFMT6_11_11_10_FLOAT = 66, + VFMT6_16_16_UNORM = 67, + VFMT6_16_16_SNORM = 68, + VFMT6_16_16_FLOAT = 69, + VFMT6_16_16_UINT = 70, + VFMT6_16_16_SINT = 71, + VFMT6_32_UNORM = 72, + VFMT6_32_SNORM = 73, + VFMT6_32_FLOAT = 74, + VFMT6_32_UINT = 75, + VFMT6_32_SINT = 76, + VFMT6_32_FIXED = 77, + VFMT6_16_16_16_UNORM = 88, + VFMT6_16_16_16_SNORM = 89, + VFMT6_16_16_16_FLOAT = 90, + VFMT6_16_16_16_UINT = 91, + VFMT6_16_16_16_SINT = 92, + VFMT6_16_16_16_16_UNORM = 96, + VFMT6_16_16_16_16_SNORM = 97, + VFMT6_16_16_16_16_FLOAT = 98, + VFMT6_16_16_16_16_UINT = 99, + VFMT6_16_16_16_16_SINT = 100, + VFMT6_32_32_UNORM = 101, + VFMT6_32_32_SNORM = 102, + VFMT6_32_32_FLOAT = 103, + VFMT6_32_32_UINT = 104, + VFMT6_32_32_SINT = 105, + VFMT6_32_32_FIXED = 106, + VFMT6_32_32_32_UNORM = 112, + VFMT6_32_32_32_SNORM = 113, + VFMT6_32_32_32_UINT = 114, + VFMT6_32_32_32_SINT = 115, + VFMT6_32_32_32_FLOAT = 116, + VFMT6_32_32_32_FIXED = 117, + VFMT6_32_32_32_32_UNORM = 128, + VFMT6_32_32_32_32_SNORM = 129, + VFMT6_32_32_32_32_FLOAT = 130, + VFMT6_32_32_32_32_UINT = 131, + VFMT6_32_32_32_32_SINT = 132, + VFMT6_32_32_32_32_FIXED = 133, +}; + +enum a6xx_tex_fmt { + TFMT6_A8_UNORM = 2, + TFMT6_8_UNORM = 3, + TFMT6_8_SNORM = 4, + TFMT6_8_UINT = 5, + TFMT6_8_SINT = 6, + TFMT6_4_4_4_4_UNORM = 8, + TFMT6_5_5_5_1_UNORM = 10, + TFMT6_5_6_5_UNORM = 14, + TFMT6_8_8_UNORM = 15, + TFMT6_8_8_SNORM = 16, + TFMT6_8_8_UINT = 17, + TFMT6_8_8_SINT = 18, + TFMT6_L8_A8_UNORM = 19, + TFMT6_16_UNORM = 21, + TFMT6_16_SNORM = 22, + TFMT6_16_FLOAT = 23, + TFMT6_16_UINT = 24, + TFMT6_16_SINT = 25, + TFMT6_8_8_8_8_UNORM = 48, + TFMT6_8_8_8_UNORM = 49, + TFMT6_8_8_8_8_SNORM = 50, + TFMT6_8_8_8_8_UINT = 51, + TFMT6_8_8_8_8_SINT = 52, + TFMT6_9_9_9_E5_FLOAT = 53, + TFMT6_10_10_10_2_UNORM = 54, + TFMT6_10_10_10_2_UINT = 58, + TFMT6_11_11_10_FLOAT = 66, + TFMT6_16_16_UNORM = 67, + TFMT6_16_16_SNORM = 68, + TFMT6_16_16_FLOAT = 69, + TFMT6_16_16_UINT = 70, + TFMT6_16_16_SINT = 71, + TFMT6_32_FLOAT = 74, + TFMT6_32_UINT = 75, + TFMT6_32_SINT = 76, + TFMT6_16_16_16_16_UNORM = 96, + TFMT6_16_16_16_16_SNORM = 97, + TFMT6_16_16_16_16_FLOAT = 98, + TFMT6_16_16_16_16_UINT = 99, + TFMT6_16_16_16_16_SINT = 100, + TFMT6_32_32_FLOAT = 103, + TFMT6_32_32_UINT = 104, + TFMT6_32_32_SINT = 105, + TFMT6_32_32_32_UINT = 114, + TFMT6_32_32_32_SINT = 115, + TFMT6_32_32_32_FLOAT = 116, + TFMT6_32_32_32_32_FLOAT = 130, + TFMT6_32_32_32_32_UINT = 131, + TFMT6_32_32_32_32_SINT = 132, + TFMT6_X8Z24_UNORM = 160, + TFMT6_ETC2_RG11_UNORM = 171, + TFMT6_ETC2_RG11_SNORM = 172, + TFMT6_ETC2_R11_UNORM = 173, + TFMT6_ETC2_R11_SNORM = 174, + TFMT6_ETC1 = 175, + TFMT6_ETC2_RGB8 = 176, + TFMT6_ETC2_RGBA8 = 177, + TFMT6_ETC2_RGB8A1 = 178, + TFMT6_DXT1 = 179, + TFMT6_DXT3 = 180, + TFMT6_DXT5 = 181, + TFMT6_RGTC1_UNORM = 183, + TFMT6_RGTC1_SNORM = 184, + TFMT6_RGTC2_UNORM = 187, + TFMT6_RGTC2_SNORM = 188, + TFMT6_BPTC_UFLOAT = 190, + TFMT6_BPTC_FLOAT = 191, + TFMT6_BPTC = 192, + TFMT6_ASTC_4x4 = 193, + TFMT6_ASTC_5x4 = 194, + TFMT6_ASTC_5x5 = 195, + TFMT6_ASTC_6x5 = 196, + TFMT6_ASTC_6x6 = 197, + TFMT6_ASTC_8x5 = 198, + TFMT6_ASTC_8x6 = 199, + TFMT6_ASTC_8x8 = 200, + TFMT6_ASTC_10x5 = 201, + TFMT6_ASTC_10x6 = 202, + TFMT6_ASTC_10x8 = 203, + TFMT6_ASTC_10x10 = 204, + TFMT6_ASTC_12x10 = 205, + TFMT6_ASTC_12x12 = 206, +}; + +enum a6xx_tex_fetchsize { + TFETCH6_1_BYTE = 0, + TFETCH6_2_BYTE = 1, + TFETCH6_4_BYTE = 2, + TFETCH6_8_BYTE = 3, + TFETCH6_16_BYTE = 4, +}; + +enum a6xx_depth_format { + DEPTH6_NONE = 0, + DEPTH6_16 = 1, + DEPTH6_24_8 = 2, + DEPTH6_32 = 4, +}; + +enum a6xx_cp_perfcounter_select { + PERF_CP_ALWAYS_COUNT = 0, +}; + +enum a6xx_tex_filter { + A6XX_TEX_NEAREST = 0, + A6XX_TEX_LINEAR = 1, + A6XX_TEX_ANISO = 2, +}; + +enum a6xx_tex_clamp { + A6XX_TEX_REPEAT = 0, + A6XX_TEX_CLAMP_TO_EDGE = 1, + A6XX_TEX_MIRROR_REPEAT = 2, + A6XX_TEX_CLAMP_TO_BORDER = 3, + A6XX_TEX_MIRROR_CLAMP = 4, +}; + +enum a6xx_tex_aniso { + A6XX_TEX_ANISO_1 = 0, + A6XX_TEX_ANISO_2 = 1, + A6XX_TEX_ANISO_4 = 2, + A6XX_TEX_ANISO_8 = 3, + A6XX_TEX_ANISO_16 = 4, +}; + +enum a6xx_tex_swiz { + A6XX_TEX_X = 0, + A6XX_TEX_Y = 1, + A6XX_TEX_Z = 2, + A6XX_TEX_W = 3, + A6XX_TEX_ZERO = 4, + A6XX_TEX_ONE = 5, +}; + +enum a6xx_tex_type { + A6XX_TEX_1D = 0, + A6XX_TEX_2D = 1, + A6XX_TEX_CUBE = 2, + A6XX_TEX_3D = 3, +}; + +#define A6XX_RBBM_INT_0_MASK_RBBM_GPU_IDLE 0x00000001 +#define A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR 0x00000002 +#define A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW 0x00000040 +#define A6XX_RBBM_INT_0_MASK_RBBM_GPC_ERROR 0x00000080 +#define A6XX_RBBM_INT_0_MASK_CP_SW 0x00000100 +#define A6XX_RBBM_INT_0_MASK_CP_HW_ERROR 0x00000200 +#define A6XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_DEPTH_TS 0x00000400 +#define A6XX_RBBM_INT_0_MASK_CP_CCU_FLUSH_COLOR_TS 0x00000800 +#define A6XX_RBBM_INT_0_MASK_CP_CCU_RESOLVE_TS 0x00001000 +#define A6XX_RBBM_INT_0_MASK_CP_IB2 0x00002000 +#define A6XX_RBBM_INT_0_MASK_CP_IB1 0x00004000 +#define A6XX_RBBM_INT_0_MASK_CP_RB 0x00008000 +#define A6XX_RBBM_INT_0_MASK_CP_RB_DONE_TS 0x00020000 +#define A6XX_RBBM_INT_0_MASK_CP_WT_DONE_TS 0x00040000 +#define A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS 0x00100000 +#define A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW 0x00400000 +#define A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT 0x00800000 +#define A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS 0x01000000 +#define A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR 0x02000000 +#define A6XX_RBBM_INT_0_MASK_DEBBUS_INTR_0 0x04000000 +#define A6XX_RBBM_INT_0_MASK_DEBBUS_INTR_1 0x08000000 +#define A6XX_RBBM_INT_0_MASK_ISDB_CPU_IRQ 0x40000000 +#define A6XX_RBBM_INT_0_MASK_ISDB_UNDER_DEBUG 0x80000000 +#define A6XX_CP_INT_CP_OPCODE_ERROR 0x00000001 +#define A6XX_CP_INT_CP_UCODE_ERROR 0x00000002 +#define A6XX_CP_INT_CP_HW_FAULT_ERROR 0x00000004 +#define A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR 0x00000010 +#define A6XX_CP_INT_CP_AHB_ERROR 0x00000020 +#define A6XX_CP_INT_CP_VSD_PARITY_ERROR 0x00000040 +#define A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR 0x00000080 +#define REG_A6XX_CP_RB_BASE 0x00000800 + +#define REG_A6XX_CP_RB_BASE_HI 0x00000801 + +#define REG_A6XX_CP_RB_CNTL 0x00000802 + +#define REG_A6XX_CP_RB_RPTR_ADDR_LO 0x00000804 + +#define REG_A6XX_CP_RB_RPTR_ADDR_HI 0x00000805 + +#define REG_A6XX_CP_RB_RPTR 0x00000806 + +#define REG_A6XX_CP_RB_WPTR 0x00000807 + +#define REG_A6XX_CP_SQE_CNTL 0x00000808 + +#define REG_A6XX_CP_HW_FAULT 0x00000821 + +#define REG_A6XX_CP_INTERRUPT_STATUS 0x00000823 + +#define REG_A6XX_CP_PROTECT_STATUS 0x00000824 + +#define REG_A6XX_CP_SQE_INSTR_BASE_LO 0x00000830 + +#define REG_A6XX_CP_SQE_INSTR_BASE_HI 0x00000831 + +#define REG_A6XX_CP_MISC_CNTL 0x00000840 + +#define REG_A6XX_CP_ROQ_THRESHOLDS_1 0x000008c1 + +#define REG_A6XX_CP_ROQ_THRESHOLDS_2 0x000008c2 + +#define REG_A6XX_CP_MEM_POOL_SIZE 0x000008c3 + +#define REG_A6XX_CP_CHICKEN_DBG 0x00000841 + +#define REG_A6XX_CP_ADDR_MODE_CNTL 0x00000842 + +#define REG_A6XX_CP_DBG_ECO_CNTL 0x00000843 + +#define REG_A6XX_CP_PROTECT_CNTL 0x0000084f + +static inline uint32_t REG_A6XX_CP_SCRATCH(uint32_t i0) { return 0x00000883 + 0x1*i0; } + +static inline uint32_t REG_A6XX_CP_SCRATCH_REG(uint32_t i0) { return 0x00000883 + 0x1*i0; } + +static inline uint32_t REG_A6XX_CP_PROTECT(uint32_t i0) { return 0x00000850 + 0x1*i0; } + +static inline uint32_t REG_A6XX_CP_PROTECT_REG(uint32_t i0) { return 0x00000850 + 0x1*i0; } +#define A6XX_CP_PROTECT_REG_BASE_ADDR__MASK 0x0003ffff +#define A6XX_CP_PROTECT_REG_BASE_ADDR__SHIFT 0 +static inline uint32_t A6XX_CP_PROTECT_REG_BASE_ADDR(uint32_t val) +{ + return ((val) << A6XX_CP_PROTECT_REG_BASE_ADDR__SHIFT) & A6XX_CP_PROTECT_REG_BASE_ADDR__MASK; +} +#define A6XX_CP_PROTECT_REG_MASK_LEN__MASK 0x7ffc0000 +#define A6XX_CP_PROTECT_REG_MASK_LEN__SHIFT 18 +static inline uint32_t A6XX_CP_PROTECT_REG_MASK_LEN(uint32_t val) +{ + return ((val) << A6XX_CP_PROTECT_REG_MASK_LEN__SHIFT) & A6XX_CP_PROTECT_REG_MASK_LEN__MASK; +} +#define A6XX_CP_PROTECT_REG_READ 0x80000000 + +#define REG_A6XX_CP_CONTEXT_SWITCH_CNTL 0x000008a0 + +#define REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_LO 0x000008a1 + +#define REG_A6XX_CP_CONTEXT_SWITCH_SMMU_INFO_HI 0x000008a2 + +#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_LO 0x000008a3 + +#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_NON_SECURE_RESTORE_ADDR_HI 0x000008a4 + +#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_LO 0x000008a5 + +#define REG_A6XX_CP_CONTEXT_SWITCH_PRIV_SECURE_RESTORE_ADDR_HI 0x000008a6 + +#define REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_LO 0x000008a7 + +#define REG_A6XX_CP_CONTEXT_SWITCH_NON_PRIV_RESTORE_ADDR_HI 0x000008a8 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_0 0x000008d0 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_1 0x000008d1 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_2 0x000008d2 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_3 0x000008d3 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_4 0x000008d4 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_5 0x000008d5 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_6 0x000008d6 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_7 0x000008d7 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_8 0x000008d8 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_9 0x000008d9 + +#define REG_A6XX_CP_PERFCTR_CP_SEL_10 0x000008da + +#define REG_A6XX_CP_PERFCTR_CP_SEL_11 0x000008db + +#define REG_A6XX_CP_PERFCTR_CP_SEL_12 0x000008dc + +#define REG_A6XX_CP_PERFCTR_CP_SEL_13 0x000008dd + +#define REG_A6XX_CP_CRASH_SCRIPT_BASE_LO 0x00000900 + +#define REG_A6XX_CP_CRASH_SCRIPT_BASE_HI 0x00000901 + +#define REG_A6XX_CP_CRASH_DUMP_CNTL 0x00000902 + +#define REG_A6XX_CP_CRASH_DUMP_STATUS 0x00000903 + +#define REG_A6XX_CP_SQE_STAT_ADDR 0x00000908 + +#define REG_A6XX_CP_SQE_STAT_DATA 0x00000909 + +#define REG_A6XX_CP_DRAW_STATE_ADDR 0x0000090a + +#define REG_A6XX_CP_DRAW_STATE_DATA 0x0000090b + +#define REG_A6XX_CP_ROQ_DBG_ADDR 0x0000090c + +#define REG_A6XX_CP_ROQ_DBG_DATA 0x0000090d + +#define REG_A6XX_CP_MEM_POOL_DBG_ADDR 0x0000090e + +#define REG_A6XX_CP_MEM_POOL_DBG_DATA 0x0000090f + +#define REG_A6XX_CP_SQE_UCODE_DBG_ADDR 0x00000910 + +#define REG_A6XX_CP_SQE_UCODE_DBG_DATA 0x00000911 + +#define REG_A6XX_CP_IB1_BASE 0x00000928 + +#define REG_A6XX_CP_IB1_BASE_HI 0x00000929 + +#define REG_A6XX_CP_IB1_REM_SIZE 0x0000092a + +#define REG_A6XX_CP_IB2_BASE 0x0000092b + +#define REG_A6XX_CP_IB2_BASE_HI 0x0000092c + +#define REG_A6XX_CP_IB2_REM_SIZE 0x0000092d + +#define REG_A6XX_CP_ALWAYS_ON_COUNTER_LO 0x00000980 + +#define REG_A6XX_CP_ALWAYS_ON_COUNTER_HI 0x00000981 + +#define REG_A6XX_CP_AHB_CNTL 0x0000098d + +#define REG_A6XX_CP_APERTURE_CNTL_HOST 0x00000a00 + +#define REG_A6XX_CP_APERTURE_CNTL_CD 0x00000a03 + +#define REG_A6XX_VSC_ADDR_MODE_CNTL 0x00000c01 + +#define REG_A6XX_RBBM_INT_0_STATUS 0x00000201 + +#define REG_A6XX_RBBM_STATUS 0x00000210 +#define A6XX_RBBM_STATUS_GPU_BUSY_IGN_AHB 0x00800000 +#define A6XX_RBBM_STATUS_GPU_BUSY_IGN_AHB_CP 0x00400000 +#define A6XX_RBBM_STATUS_HLSQ_BUSY 0x00200000 +#define A6XX_RBBM_STATUS_VSC_BUSY 0x00100000 +#define A6XX_RBBM_STATUS_TPL1_BUSY 0x00080000 +#define A6XX_RBBM_STATUS_SP_BUSY 0x00040000 +#define A6XX_RBBM_STATUS_UCHE_BUSY 0x00020000 +#define A6XX_RBBM_STATUS_VPC_BUSY 0x00010000 +#define A6XX_RBBM_STATUS_VFD_BUSY 0x00008000 +#define A6XX_RBBM_STATUS_TESS_BUSY 0x00004000 +#define A6XX_RBBM_STATUS_PC_VSD_BUSY 0x00002000 +#define A6XX_RBBM_STATUS_PC_DCALL_BUSY 0x00001000 +#define A6XX_RBBM_STATUS_COM_DCOM_BUSY 0x00000800 +#define A6XX_RBBM_STATUS_LRZ_BUSY 0x00000400 +#define A6XX_RBBM_STATUS_A2D_BUSY 0x00000200 +#define A6XX_RBBM_STATUS_CCU_BUSY 0x00000100 +#define A6XX_RBBM_STATUS_RB_BUSY 0x00000080 +#define A6XX_RBBM_STATUS_RAS_BUSY 0x00000040 +#define A6XX_RBBM_STATUS_TSE_BUSY 0x00000020 +#define A6XX_RBBM_STATUS_VBIF_BUSY 0x00000010 +#define A6XX_RBBM_STATUS_GFX_DBGC_BUSY 0x00000008 +#define A6XX_RBBM_STATUS_CP_BUSY 0x00000004 +#define A6XX_RBBM_STATUS_CP_AHB_BUSY_CP_MASTER 0x00000002 +#define A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER 0x00000001 + +#define REG_A6XX_RBBM_STATUS3 0x00000213 + +#define REG_A6XX_RBBM_VBIF_GX_RESET_STATUS 0x00000215 + +#define REG_A6XX_RBBM_PERFCTR_CP_0_LO 0x00000400 + +#define REG_A6XX_RBBM_PERFCTR_CP_0_HI 0x00000401 + +#define REG_A6XX_RBBM_PERFCTR_CP_1_LO 0x00000402 + +#define REG_A6XX_RBBM_PERFCTR_CP_1_HI 0x00000403 + +#define REG_A6XX_RBBM_PERFCTR_CP_2_LO 0x00000404 + +#define REG_A6XX_RBBM_PERFCTR_CP_2_HI 0x00000405 + +#define REG_A6XX_RBBM_PERFCTR_CP_3_LO 0x00000406 + +#define REG_A6XX_RBBM_PERFCTR_CP_3_HI 0x00000407 + +#define REG_A6XX_RBBM_PERFCTR_CP_4_LO 0x00000408 + +#define REG_A6XX_RBBM_PERFCTR_CP_4_HI 0x00000409 + +#define REG_A6XX_RBBM_PERFCTR_CP_5_LO 0x0000040a + +#define REG_A6XX_RBBM_PERFCTR_CP_5_HI 0x0000040b + +#define REG_A6XX_RBBM_PERFCTR_CP_6_LO 0x0000040c + +#define REG_A6XX_RBBM_PERFCTR_CP_6_HI 0x0000040d + +#define REG_A6XX_RBBM_PERFCTR_CP_7_LO 0x0000040e + +#define REG_A6XX_RBBM_PERFCTR_CP_7_HI 0x0000040f + +#define REG_A6XX_RBBM_PERFCTR_CP_8_LO 0x00000410 + +#define REG_A6XX_RBBM_PERFCTR_CP_8_HI 0x00000411 + +#define REG_A6XX_RBBM_PERFCTR_CP_9_LO 0x00000412 + +#define REG_A6XX_RBBM_PERFCTR_CP_9_HI 0x00000413 + +#define REG_A6XX_RBBM_PERFCTR_CP_10_LO 0x00000414 + +#define REG_A6XX_RBBM_PERFCTR_CP_10_HI 0x00000415 + +#define REG_A6XX_RBBM_PERFCTR_CP_11_LO 0x00000416 + +#define REG_A6XX_RBBM_PERFCTR_CP_11_HI 0x00000417 + +#define REG_A6XX_RBBM_PERFCTR_CP_12_LO 0x00000418 + +#define REG_A6XX_RBBM_PERFCTR_CP_12_HI 0x00000419 + +#define REG_A6XX_RBBM_PERFCTR_CP_13_LO 0x0000041a + +#define REG_A6XX_RBBM_PERFCTR_CP_13_HI 0x0000041b + +#define REG_A6XX_RBBM_PERFCTR_RBBM_0_LO 0x0000041c + +#define REG_A6XX_RBBM_PERFCTR_RBBM_0_HI 0x0000041d + +#define REG_A6XX_RBBM_PERFCTR_RBBM_1_LO 0x0000041e + +#define REG_A6XX_RBBM_PERFCTR_RBBM_1_HI 0x0000041f + +#define REG_A6XX_RBBM_PERFCTR_RBBM_2_LO 0x00000420 + +#define REG_A6XX_RBBM_PERFCTR_RBBM_2_HI 0x00000421 + +#define REG_A6XX_RBBM_PERFCTR_RBBM_3_LO 0x00000422 + +#define REG_A6XX_RBBM_PERFCTR_RBBM_3_HI 0x00000423 + +#define REG_A6XX_RBBM_PERFCTR_PC_0_LO 0x00000424 + +#define REG_A6XX_RBBM_PERFCTR_PC_0_HI 0x00000425 + +#define REG_A6XX_RBBM_PERFCTR_PC_1_LO 0x00000426 + +#define REG_A6XX_RBBM_PERFCTR_PC_1_HI 0x00000427 + +#define REG_A6XX_RBBM_PERFCTR_PC_2_LO 0x00000428 + +#define REG_A6XX_RBBM_PERFCTR_PC_2_HI 0x00000429 + +#define REG_A6XX_RBBM_PERFCTR_PC_3_LO 0x0000042a + +#define REG_A6XX_RBBM_PERFCTR_PC_3_HI 0x0000042b + +#define REG_A6XX_RBBM_PERFCTR_PC_4_LO 0x0000042c + +#define REG_A6XX_RBBM_PERFCTR_PC_4_HI 0x0000042d + +#define REG_A6XX_RBBM_PERFCTR_PC_5_LO 0x0000042e + +#define REG_A6XX_RBBM_PERFCTR_PC_5_HI 0x0000042f + +#define REG_A6XX_RBBM_PERFCTR_PC_6_LO 0x00000430 + +#define REG_A6XX_RBBM_PERFCTR_PC_6_HI 0x00000431 + +#define REG_A6XX_RBBM_PERFCTR_PC_7_LO 0x00000432 + +#define REG_A6XX_RBBM_PERFCTR_PC_7_HI 0x00000433 + +#define REG_A6XX_RBBM_PERFCTR_VFD_0_LO 0x00000434 + +#define REG_A6XX_RBBM_PERFCTR_VFD_0_HI 0x00000435 + +#define REG_A6XX_RBBM_PERFCTR_VFD_1_LO 0x00000436 + +#define REG_A6XX_RBBM_PERFCTR_VFD_1_HI 0x00000437 + +#define REG_A6XX_RBBM_PERFCTR_VFD_2_LO 0x00000438 + +#define REG_A6XX_RBBM_PERFCTR_VFD_2_HI 0x00000439 + +#define REG_A6XX_RBBM_PERFCTR_VFD_3_LO 0x0000043a + +#define REG_A6XX_RBBM_PERFCTR_VFD_3_HI 0x0000043b + +#define REG_A6XX_RBBM_PERFCTR_VFD_4_LO 0x0000043c + +#define REG_A6XX_RBBM_PERFCTR_VFD_4_HI 0x0000043d + +#define REG_A6XX_RBBM_PERFCTR_VFD_5_LO 0x0000043e + +#define REG_A6XX_RBBM_PERFCTR_VFD_5_HI 0x0000043f + +#define REG_A6XX_RBBM_PERFCTR_VFD_6_LO 0x00000440 + +#define REG_A6XX_RBBM_PERFCTR_VFD_6_HI 0x00000441 + +#define REG_A6XX_RBBM_PERFCTR_VFD_7_LO 0x00000442 + +#define REG_A6XX_RBBM_PERFCTR_VFD_7_HI 0x00000443 + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_0_LO 0x00000444 + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_0_HI 0x00000445 + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_1_LO 0x00000446 + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_1_HI 0x00000447 + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_2_LO 0x00000448 + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_2_HI 0x00000449 + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_3_LO 0x0000044a + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_3_HI 0x0000044b + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_4_LO 0x0000044c + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_4_HI 0x0000044d + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_5_LO 0x0000044e + +#define REG_A6XX_RBBM_PERFCTR_HLSQ_5_HI 0x0000044f + +#define REG_A6XX_RBBM_PERFCTR_VPC_0_LO 0x00000450 + +#define REG_A6XX_RBBM_PERFCTR_VPC_0_HI 0x00000451 + +#define REG_A6XX_RBBM_PERFCTR_VPC_1_LO 0x00000452 + +#define REG_A6XX_RBBM_PERFCTR_VPC_1_HI 0x00000453 + +#define REG_A6XX_RBBM_PERFCTR_VPC_2_LO 0x00000454 + +#define REG_A6XX_RBBM_PERFCTR_VPC_2_HI 0x00000455 + +#define REG_A6XX_RBBM_PERFCTR_VPC_3_LO 0x00000456 + +#define REG_A6XX_RBBM_PERFCTR_VPC_3_HI 0x00000457 + +#define REG_A6XX_RBBM_PERFCTR_VPC_4_LO 0x00000458 + +#define REG_A6XX_RBBM_PERFCTR_VPC_4_HI 0x00000459 + +#define REG_A6XX_RBBM_PERFCTR_VPC_5_LO 0x0000045a + +#define REG_A6XX_RBBM_PERFCTR_VPC_5_HI 0x0000045b + +#define REG_A6XX_RBBM_PERFCTR_CCU_0_LO 0x0000045c + +#define REG_A6XX_RBBM_PERFCTR_CCU_0_HI 0x0000045d + +#define REG_A6XX_RBBM_PERFCTR_CCU_1_LO 0x0000045e + +#define REG_A6XX_RBBM_PERFCTR_CCU_1_HI 0x0000045f + +#define REG_A6XX_RBBM_PERFCTR_CCU_2_LO 0x00000460 + +#define REG_A6XX_RBBM_PERFCTR_CCU_2_HI 0x00000461 + +#define REG_A6XX_RBBM_PERFCTR_CCU_3_LO 0x00000462 + +#define REG_A6XX_RBBM_PERFCTR_CCU_3_HI 0x00000463 + +#define REG_A6XX_RBBM_PERFCTR_CCU_4_LO 0x00000464 + +#define REG_A6XX_RBBM_PERFCTR_CCU_4_HI 0x00000465 + +#define REG_A6XX_RBBM_PERFCTR_TSE_0_LO 0x00000466 + +#define REG_A6XX_RBBM_PERFCTR_TSE_0_HI 0x00000467 + +#define REG_A6XX_RBBM_PERFCTR_TSE_1_LO 0x00000468 + +#define REG_A6XX_RBBM_PERFCTR_TSE_1_HI 0x00000469 + +#define REG_A6XX_RBBM_PERFCTR_TSE_2_LO 0x0000046a + +#define REG_A6XX_RBBM_PERFCTR_CCU_4_HI 0x00000465 + +#define REG_A6XX_RBBM_PERFCTR_TSE_0_LO 0x00000466 + +#define REG_A6XX_RBBM_PERFCTR_TSE_0_HI 0x00000467 + +#define REG_A6XX_RBBM_PERFCTR_TSE_1_LO 0x00000468 + +#define REG_A6XX_RBBM_PERFCTR_TSE_1_HI 0x00000469 + +#define REG_A6XX_RBBM_PERFCTR_TSE_2_LO 0x0000046a + +#define REG_A6XX_RBBM_PERFCTR_TSE_2_HI 0x0000046b + +#define REG_A6XX_RBBM_PERFCTR_TSE_3_LO 0x0000046c + +#define REG_A6XX_RBBM_PERFCTR_TSE_3_HI 0x0000046d + +#define REG_A6XX_RBBM_PERFCTR_RAS_0_LO 0x0000046e + +#define REG_A6XX_RBBM_PERFCTR_RAS_0_HI 0x0000046f + +#define REG_A6XX_RBBM_PERFCTR_RAS_1_LO 0x00000470 + +#define REG_A6XX_RBBM_PERFCTR_RAS_1_HI 0x00000471 + +#define REG_A6XX_RBBM_PERFCTR_RAS_2_LO 0x00000472 + +#define REG_A6XX_RBBM_PERFCTR_RAS_2_HI 0x00000473 + +#define REG_A6XX_RBBM_PERFCTR_RAS_3_LO 0x00000474 + +#define REG_A6XX_RBBM_PERFCTR_RAS_3_HI 0x00000475 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_0_LO 0x00000476 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_0_HI 0x00000477 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_1_LO 0x00000478 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_1_HI 0x00000479 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_2_LO 0x0000047a + +#define REG_A6XX_RBBM_PERFCTR_UCHE_2_HI 0x0000047b + +#define REG_A6XX_RBBM_PERFCTR_UCHE_3_LO 0x0000047c + +#define REG_A6XX_RBBM_PERFCTR_UCHE_3_HI 0x0000047d + +#define REG_A6XX_RBBM_PERFCTR_UCHE_4_LO 0x0000047e + +#define REG_A6XX_RBBM_PERFCTR_UCHE_4_HI 0x0000047f + +#define REG_A6XX_RBBM_PERFCTR_UCHE_5_LO 0x00000480 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_5_HI 0x00000481 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_6_LO 0x00000482 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_6_HI 0x00000483 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_7_LO 0x00000484 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_7_HI 0x00000485 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_8_LO 0x00000486 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_8_HI 0x00000487 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_9_LO 0x00000488 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_9_HI 0x00000489 + +#define REG_A6XX_RBBM_PERFCTR_UCHE_10_LO 0x0000048a + +#define REG_A6XX_RBBM_PERFCTR_UCHE_10_HI 0x0000048b + +#define REG_A6XX_RBBM_PERFCTR_UCHE_11_LO 0x0000048c + +#define REG_A6XX_RBBM_PERFCTR_UCHE_11_HI 0x0000048d + +#define REG_A6XX_RBBM_PERFCTR_TP_0_LO 0x0000048e + +#define REG_A6XX_RBBM_PERFCTR_TP_0_HI 0x0000048f + +#define REG_A6XX_RBBM_PERFCTR_TP_1_LO 0x00000490 + +#define REG_A6XX_RBBM_PERFCTR_TP_1_HI 0x00000491 + +#define REG_A6XX_RBBM_PERFCTR_TP_2_LO 0x00000492 + +#define REG_A6XX_RBBM_PERFCTR_TP_2_HI 0x00000493 + +#define REG_A6XX_RBBM_PERFCTR_TP_3_LO 0x00000494 + +#define REG_A6XX_RBBM_PERFCTR_TP_3_HI 0x00000495 + +#define REG_A6XX_RBBM_PERFCTR_TP_4_LO 0x00000496 + +#define REG_A6XX_RBBM_PERFCTR_TP_4_HI 0x00000497 + +#define REG_A6XX_RBBM_PERFCTR_TP_5_LO 0x00000498 + +#define REG_A6XX_RBBM_PERFCTR_TP_5_HI 0x00000499 + +#define REG_A6XX_RBBM_PERFCTR_TP_6_LO 0x0000049a + +#define REG_A6XX_RBBM_PERFCTR_TP_6_HI 0x0000049b + +#define REG_A6XX_RBBM_PERFCTR_TP_7_LO 0x0000049c + +#define REG_A6XX_RBBM_PERFCTR_TP_7_HI 0x0000049d + +#define REG_A6XX_RBBM_PERFCTR_TP_8_LO 0x0000049e + +#define REG_A6XX_RBBM_PERFCTR_TP_8_HI 0x0000049f + +#define REG_A6XX_RBBM_PERFCTR_TP_9_LO 0x000004a0 + +#define REG_A6XX_RBBM_PERFCTR_TP_9_HI 0x000004a1 + +#define REG_A6XX_RBBM_PERFCTR_TP_10_LO 0x000004a2 + +#define REG_A6XX_RBBM_PERFCTR_TP_10_HI 0x000004a3 + +#define REG_A6XX_RBBM_PERFCTR_TP_11_LO 0x000004a4 + +#define REG_A6XX_RBBM_PERFCTR_TP_11_HI 0x000004a5 + +#define REG_A6XX_RBBM_PERFCTR_SP_0_LO 0x000004a6 + +#define REG_A6XX_RBBM_PERFCTR_SP_0_HI 0x000004a7 + +#define REG_A6XX_RBBM_PERFCTR_SP_1_LO 0x000004a8 + +#define REG_A6XX_RBBM_PERFCTR_SP_1_HI 0x000004a9 + +#define REG_A6XX_RBBM_PERFCTR_SP_2_LO 0x000004aa + +#define REG_A6XX_RBBM_PERFCTR_SP_2_HI 0x000004ab + +#define REG_A6XX_RBBM_PERFCTR_SP_3_LO 0x000004ac + +#define REG_A6XX_RBBM_PERFCTR_SP_3_HI 0x000004ad + +#define REG_A6XX_RBBM_PERFCTR_SP_4_LO 0x000004ae + +#define REG_A6XX_RBBM_PERFCTR_SP_4_HI 0x000004af + +#define REG_A6XX_RBBM_PERFCTR_SP_5_LO 0x000004b0 + +#define REG_A6XX_RBBM_PERFCTR_SP_5_HI 0x000004b1 + +#define REG_A6XX_RBBM_PERFCTR_SP_6_LO 0x000004b2 + +#define REG_A6XX_RBBM_PERFCTR_SP_6_HI 0x000004b3 + +#define REG_A6XX_RBBM_PERFCTR_SP_7_LO 0x000004b4 + +#define REG_A6XX_RBBM_PERFCTR_SP_7_HI 0x000004b5 + +#define REG_A6XX_RBBM_PERFCTR_SP_8_LO 0x000004b6 + +#define REG_A6XX_RBBM_PERFCTR_SP_8_HI 0x000004b7 + +#define REG_A6XX_RBBM_PERFCTR_SP_9_LO 0x000004b8 + +#define REG_A6XX_RBBM_PERFCTR_SP_9_HI 0x000004b9 + +#define REG_A6XX_RBBM_PERFCTR_SP_10_LO 0x000004ba + +#define REG_A6XX_RBBM_PERFCTR_SP_10_HI 0x000004bb + +#define REG_A6XX_RBBM_PERFCTR_SP_11_LO 0x000004bc + +#define REG_A6XX_RBBM_PERFCTR_SP_11_HI 0x000004bd + +#define REG_A6XX_RBBM_PERFCTR_SP_12_LO 0x000004be + +#define REG_A6XX_RBBM_PERFCTR_SP_12_HI 0x000004bf + +#define REG_A6XX_RBBM_PERFCTR_SP_13_LO 0x000004c0 + +#define REG_A6XX_RBBM_PERFCTR_SP_13_HI 0x000004c1 + +#define REG_A6XX_RBBM_PERFCTR_SP_14_LO 0x000004c2 + +#define REG_A6XX_RBBM_PERFCTR_SP_14_HI 0x000004c3 + +#define REG_A6XX_RBBM_PERFCTR_SP_15_LO 0x000004c4 + +#define REG_A6XX_RBBM_PERFCTR_SP_15_HI 0x000004c5 + +#define REG_A6XX_RBBM_PERFCTR_SP_16_LO 0x000004c6 + +#define REG_A6XX_RBBM_PERFCTR_SP_16_HI 0x000004c7 + +#define REG_A6XX_RBBM_PERFCTR_SP_17_LO 0x000004c8 + +#define REG_A6XX_RBBM_PERFCTR_SP_17_HI 0x000004c9 + +#define REG_A6XX_RBBM_PERFCTR_SP_18_LO 0x000004ca + +#define REG_A6XX_RBBM_PERFCTR_SP_18_HI 0x000004cb + +#define REG_A6XX_RBBM_PERFCTR_SP_19_LO 0x000004cc + +#define REG_A6XX_RBBM_PERFCTR_SP_19_HI 0x000004cd + +#define REG_A6XX_RBBM_PERFCTR_SP_20_LO 0x000004ce + +#define REG_A6XX_RBBM_PERFCTR_SP_20_HI 0x000004cf + +#define REG_A6XX_RBBM_PERFCTR_SP_21_LO 0x000004d0 + +#define REG_A6XX_RBBM_PERFCTR_SP_21_HI 0x000004d1 + +#define REG_A6XX_RBBM_PERFCTR_SP_22_LO 0x000004d2 + +#define REG_A6XX_RBBM_PERFCTR_SP_22_HI 0x000004d3 + +#define REG_A6XX_RBBM_PERFCTR_SP_23_LO 0x000004d4 + +#define REG_A6XX_RBBM_PERFCTR_SP_23_HI 0x000004d5 + +#define REG_A6XX_RBBM_PERFCTR_RB_0_LO 0x000004d6 + +#define REG_A6XX_RBBM_PERFCTR_RB_0_HI 0x000004d7 + +#define REG_A6XX_RBBM_PERFCTR_RB_1_LO 0x000004d8 + +#define REG_A6XX_RBBM_PERFCTR_RB_1_HI 0x000004d9 + +#define REG_A6XX_RBBM_PERFCTR_RB_2_LO 0x000004da + +#define REG_A6XX_RBBM_PERFCTR_RB_2_HI 0x000004db + +#define REG_A6XX_RBBM_PERFCTR_RB_3_LO 0x000004dc + +#define REG_A6XX_RBBM_PERFCTR_RB_3_HI 0x000004dd + +#define REG_A6XX_RBBM_PERFCTR_RB_4_LO 0x000004de + +#define REG_A6XX_RBBM_PERFCTR_RB_4_HI 0x000004df + +#define REG_A6XX_RBBM_PERFCTR_RB_5_LO 0x000004e0 + +#define REG_A6XX_RBBM_PERFCTR_RB_5_HI 0x000004e1 + +#define REG_A6XX_RBBM_PERFCTR_RB_6_LO 0x000004e2 + +#define REG_A6XX_RBBM_PERFCTR_RB_6_HI 0x000004e3 + +#define REG_A6XX_RBBM_PERFCTR_RB_7_LO 0x000004e4 + +#define REG_A6XX_RBBM_PERFCTR_RB_7_HI 0x000004e5 + +#define REG_A6XX_RBBM_PERFCTR_VSC_0_LO 0x000004e6 + +#define REG_A6XX_RBBM_PERFCTR_VSC_0_HI 0x000004e7 + +#define REG_A6XX_RBBM_PERFCTR_VSC_1_LO 0x000004e8 + +#define REG_A6XX_RBBM_PERFCTR_VSC_1_HI 0x000004e9 + +#define REG_A6XX_RBBM_PERFCTR_LRZ_0_LO 0x000004ea + +#define REG_A6XX_RBBM_PERFCTR_LRZ_0_HI 0x000004eb + +#define REG_A6XX_RBBM_PERFCTR_LRZ_1_LO 0x000004ec + +#define REG_A6XX_RBBM_PERFCTR_LRZ_1_HI 0x000004ed + +#define REG_A6XX_RBBM_PERFCTR_LRZ_2_LO 0x000004ee + +#define REG_A6XX_RBBM_PERFCTR_LRZ_2_HI 0x000004ef + +#define REG_A6XX_RBBM_PERFCTR_LRZ_3_LO 0x000004f0 + +#define REG_A6XX_RBBM_PERFCTR_LRZ_3_HI 0x000004f1 + +#define REG_A6XX_RBBM_PERFCTR_CMP_0_LO 0x000004f2 + +#define REG_A6XX_RBBM_PERFCTR_CMP_0_HI 0x000004f3 + +#define REG_A6XX_RBBM_PERFCTR_CMP_1_LO 0x000004f4 + +#define REG_A6XX_RBBM_PERFCTR_CMP_1_HI 0x000004f5 + +#define REG_A6XX_RBBM_PERFCTR_CMP_2_LO 0x000004f6 + +#define REG_A6XX_RBBM_PERFCTR_CMP_2_HI 0x000004f7 + +#define REG_A6XX_RBBM_PERFCTR_CMP_3_LO 0x000004f8 + +#define REG_A6XX_RBBM_PERFCTR_CMP_3_HI 0x000004f9 + +#define REG_A6XX_RBBM_PERFCTR_CNTL 0x00000500 + +#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD0 0x00000501 + +#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD1 0x00000502 + +#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD2 0x00000503 + +#define REG_A6XX_RBBM_PERFCTR_LOAD_CMD3 0x00000504 + +#define REG_A6XX_RBBM_PERFCTR_LOAD_VALUE_LO 0x00000505 + +#define REG_A6XX_RBBM_PERFCTR_LOAD_VALUE_HI 0x00000506 + +#define REG_A6XX_RBBM_PERFCTR_RBBM_SEL_0 0x00000507 + +#define REG_A6XX_RBBM_PERFCTR_RBBM_SEL_1 0x00000508 + +#define REG_A6XX_RBBM_PERFCTR_RBBM_SEL_2 0x00000509 + +#define REG_A6XX_RBBM_PERFCTR_RBBM_SEL_3 0x0000050a + +#define REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED 0x0000050b + +#define REG_A6XX_RBBM_ISDB_CNT 0x00000533 + +#define REG_A6XX_RBBM_SECVID_TRUST_CNTL 0x0000f400 + +#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO 0x0000f800 + +#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI 0x0000f801 + +#define REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE 0x0000f802 + +#define REG_A6XX_RBBM_SECVID_TSB_CNTL 0x0000f803 + +#define REG_A6XX_RBBM_SECVID_TSB_ADDR_MODE_CNTL 0x0000f810 + +#define REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL 0x00000010 + +#define REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL 0x0000001f + +#define REG_A6XX_RBBM_INT_CLEAR_CMD 0x00000037 + +#define REG_A6XX_RBBM_INT_0_MASK 0x00000038 + +#define REG_A6XX_RBBM_SP_HYST_CNT 0x00000042 + +#define REG_A6XX_RBBM_SW_RESET_CMD 0x00000043 + +#define REG_A6XX_RBBM_RAC_THRESHOLD_CNT 0x00000044 + +#define REG_A6XX_RBBM_BLOCK_SW_RESET_CMD 0x00000045 + +#define REG_A6XX_RBBM_BLOCK_SW_RESET_CMD2 0x00000046 + +#define REG_A6XX_RBBM_CLOCK_CNTL 0x000000ae + +#define REG_A6XX_RBBM_CLOCK_CNTL_SP0 0x000000b0 + +#define REG_A6XX_RBBM_CLOCK_CNTL_SP1 0x000000b1 + +#define REG_A6XX_RBBM_CLOCK_CNTL_SP2 0x000000b2 + +#define REG_A6XX_RBBM_CLOCK_CNTL_SP3 0x000000b3 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_SP0 0x000000b4 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_SP1 0x000000b5 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_SP2 0x000000b6 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_SP3 0x000000b7 + +#define REG_A6XX_RBBM_CLOCK_DELAY_SP0 0x000000b8 + +#define REG_A6XX_RBBM_CLOCK_DELAY_SP1 0x000000b9 + +#define REG_A6XX_RBBM_CLOCK_DELAY_SP2 0x000000ba + +#define REG_A6XX_RBBM_CLOCK_DELAY_SP3 0x000000bb + +#define REG_A6XX_RBBM_CLOCK_HYST_SP0 0x000000bc + +#define REG_A6XX_RBBM_CLOCK_HYST_SP1 0x000000bd + +#define REG_A6XX_RBBM_CLOCK_HYST_SP2 0x000000be + +#define REG_A6XX_RBBM_CLOCK_HYST_SP3 0x000000bf + +#define REG_A6XX_RBBM_CLOCK_CNTL_TP0 0x000000c0 + +#define REG_A6XX_RBBM_CLOCK_CNTL_TP1 0x000000c1 + +#define REG_A6XX_RBBM_CLOCK_CNTL_TP2 0x000000c2 + +#define REG_A6XX_RBBM_CLOCK_CNTL_TP3 0x000000c3 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_TP0 0x000000c4 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_TP1 0x000000c5 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_TP2 0x000000c6 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_TP3 0x000000c7 + +#define REG_A6XX_RBBM_CLOCK_CNTL3_TP0 0x000000c8 + +#define REG_A6XX_RBBM_CLOCK_CNTL3_TP1 0x000000c9 + +#define REG_A6XX_RBBM_CLOCK_CNTL3_TP2 0x000000ca + +#define REG_A6XX_RBBM_CLOCK_CNTL3_TP3 0x000000cb + +#define REG_A6XX_RBBM_CLOCK_CNTL4_TP0 0x000000cc + +#define REG_A6XX_RBBM_CLOCK_CNTL4_TP1 0x000000cd + +#define REG_A6XX_RBBM_CLOCK_CNTL4_TP2 0x000000ce + +#define REG_A6XX_RBBM_CLOCK_CNTL4_TP3 0x000000cf + +#define REG_A6XX_RBBM_CLOCK_DELAY_TP0 0x000000d0 + +#define REG_A6XX_RBBM_CLOCK_DELAY_TP1 0x000000d1 + +#define REG_A6XX_RBBM_CLOCK_DELAY_TP2 0x000000d2 + +#define REG_A6XX_RBBM_CLOCK_DELAY_TP3 0x000000d3 + +#define REG_A6XX_RBBM_CLOCK_DELAY2_TP0 0x000000d4 + +#define REG_A6XX_RBBM_CLOCK_DELAY2_TP1 0x000000d5 + +#define REG_A6XX_RBBM_CLOCK_DELAY2_TP2 0x000000d6 + +#define REG_A6XX_RBBM_CLOCK_DELAY2_TP3 0x000000d7 + +#define REG_A6XX_RBBM_CLOCK_DELAY3_TP0 0x000000d8 + +#define REG_A6XX_RBBM_CLOCK_DELAY3_TP1 0x000000d9 + +#define REG_A6XX_RBBM_CLOCK_DELAY3_TP2 0x000000da + +#define REG_A6XX_RBBM_CLOCK_DELAY3_TP3 0x000000db + +#define REG_A6XX_RBBM_CLOCK_DELAY4_TP0 0x000000dc + +#define REG_A6XX_RBBM_CLOCK_DELAY4_TP1 0x000000dd + +#define REG_A6XX_RBBM_CLOCK_DELAY4_TP2 0x000000de + +#define REG_A6XX_RBBM_CLOCK_DELAY4_TP3 0x000000df + +#define REG_A6XX_RBBM_CLOCK_HYST_TP0 0x000000e0 + +#define REG_A6XX_RBBM_CLOCK_HYST_TP1 0x000000e1 + +#define REG_A6XX_RBBM_CLOCK_HYST_TP2 0x000000e2 + +#define REG_A6XX_RBBM_CLOCK_HYST_TP3 0x000000e3 + +#define REG_A6XX_RBBM_CLOCK_HYST2_TP0 0x000000e4 + +#define REG_A6XX_RBBM_CLOCK_HYST2_TP1 0x000000e5 + +#define REG_A6XX_RBBM_CLOCK_HYST2_TP2 0x000000e6 + +#define REG_A6XX_RBBM_CLOCK_HYST2_TP3 0x000000e7 + +#define REG_A6XX_RBBM_CLOCK_HYST3_TP0 0x000000e8 + +#define REG_A6XX_RBBM_CLOCK_HYST3_TP1 0x000000e9 + +#define REG_A6XX_RBBM_CLOCK_HYST3_TP2 0x000000ea + +#define REG_A6XX_RBBM_CLOCK_HYST3_TP3 0x000000eb + +#define REG_A6XX_RBBM_CLOCK_HYST4_TP0 0x000000ec + +#define REG_A6XX_RBBM_CLOCK_HYST4_TP1 0x000000ed + +#define REG_A6XX_RBBM_CLOCK_HYST4_TP2 0x000000ee + +#define REG_A6XX_RBBM_CLOCK_HYST4_TP3 0x000000ef + +#define REG_A6XX_RBBM_CLOCK_CNTL_RB0 0x000000f0 + +#define REG_A6XX_RBBM_CLOCK_CNTL_RB1 0x000000f1 + +#define REG_A6XX_RBBM_CLOCK_CNTL_RB2 0x000000f2 + +#define REG_A6XX_RBBM_CLOCK_CNTL_RB3 0x000000f3 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_RB0 0x000000f4 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_RB1 0x000000f5 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_RB2 0x000000f6 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_RB3 0x000000f7 + +#define REG_A6XX_RBBM_CLOCK_CNTL_CCU0 0x000000f8 + +#define REG_A6XX_RBBM_CLOCK_CNTL_CCU1 0x000000f9 + +#define REG_A6XX_RBBM_CLOCK_CNTL_CCU2 0x000000fa + +#define REG_A6XX_RBBM_CLOCK_CNTL_CCU3 0x000000fb + +#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0 0x00000100 + +#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1 0x00000101 + +#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2 0x00000102 + +#define REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3 0x00000103 + +#define REG_A6XX_RBBM_CLOCK_CNTL_RAC 0x00000104 + +#define REG_A6XX_RBBM_CLOCK_CNTL2_RAC 0x00000105 + +#define REG_A6XX_RBBM_CLOCK_DELAY_RAC 0x00000106 + +#define REG_A6XX_RBBM_CLOCK_HYST_RAC 0x00000107 + +#define REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM 0x00000108 + +#define REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM 0x00000109 + +#define REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM 0x0000010a + +#define REG_A6XX_RBBM_CLOCK_CNTL_UCHE 0x0000010b + +#define REG_A6XX_RBBM_CLOCK_CNTL2_UCHE 0x0000010c + +#define REG_A6XX_RBBM_CLOCK_CNTL3_UCHE 0x0000010d + +#define REG_A6XX_RBBM_CLOCK_CNTL4_UCHE 0x0000010e + +#define REG_A6XX_RBBM_CLOCK_DELAY_UCHE 0x0000010f + +#define REG_A6XX_RBBM_CLOCK_HYST_UCHE 0x00000110 + +#define REG_A6XX_RBBM_CLOCK_MODE_VFD 0x00000111 + +#define REG_A6XX_RBBM_CLOCK_DELAY_VFD 0x00000112 + +#define REG_A6XX_RBBM_CLOCK_HYST_VFD 0x00000113 + +#define REG_A6XX_RBBM_CLOCK_MODE_GPC 0x00000114 + +#define REG_A6XX_RBBM_CLOCK_DELAY_GPC 0x00000115 + +#define REG_A6XX_RBBM_CLOCK_HYST_GPC 0x00000116 + +#define REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2 0x00000117 + +#define REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX 0x00000118 + +#define REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX 0x00000119 + +#define REG_A6XX_RBBM_CLOCK_HYST_GMU_GX 0x0000011a + +#define REG_A6XX_RBBM_CLOCK_MODE_HLSQ 0x0000011b + +#define REG_A6XX_RBBM_CLOCK_DELAY_HLSQ 0x0000011c + +#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_A 0x00000600 + +#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_B 0x00000601 + +#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_C 0x00000602 + +#define REG_A6XX_DBGC_CFG_DBGBUS_SEL_D 0x00000603 +#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK 0x000000ff +#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT 0 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT) & A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK 0x0000ff00 +#define A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT 8 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT) & A6XX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK; +} + +#define REG_A6XX_DBGC_CFG_DBGBUS_CNTLT 0x00000604 +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK 0x0000003f +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT 0 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK 0x00007000 +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT 12 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK 0xf0000000 +#define A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT 28 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK; +} + +#define REG_A6XX_DBGC_CFG_DBGBUS_CNTLM 0x00000605 +#define A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK 0x0f000000 +#define A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT 24 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT) & A6XX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK; +} + +#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_0 0x00000608 + +#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_1 0x00000609 + +#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_2 0x0000060a + +#define REG_A6XX_DBGC_CFG_DBGBUS_IVTL_3 0x0000060b + +#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_0 0x0000060c + +#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_1 0x0000060d + +#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_2 0x0000060e + +#define REG_A6XX_DBGC_CFG_DBGBUS_MASKL_3 0x0000060f + +#define REG_A6XX_DBGC_CFG_DBGBUS_BYTEL_0 0x00000610 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK 0x0000000f +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT 0 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK 0x000000f0 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT 4 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK 0x00000f00 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT 8 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK 0x0000f000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT 12 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK 0x000f0000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT 16 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK 0x00f00000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT 20 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK 0x0f000000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT 24 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK 0xf0000000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT 28 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK; +} + +#define REG_A6XX_DBGC_CFG_DBGBUS_BYTEL_1 0x00000611 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK 0x0000000f +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT 0 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK 0x000000f0 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT 4 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK 0x00000f00 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT 8 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK 0x0000f000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT 12 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK 0x000f0000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT 16 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK 0x00f00000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT 20 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK 0x0f000000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT 24 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK; +} +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK 0xf0000000 +#define A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT 28 +static inline uint32_t A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15(uint32_t val) +{ + return ((val) << A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT) & A6XX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK; +} + +#define REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF1 0x0000062f + +#define REG_A6XX_DBGC_CFG_DBGBUS_TRACE_BUF2 0x00000630 + +#define REG_A6XX_VSC_PERFCTR_VSC_SEL_0 0x00000cd8 + +#define REG_A6XX_VSC_PERFCTR_VSC_SEL_1 0x00000cd9 + +#define REG_A6XX_GRAS_ADDR_MODE_CNTL 0x00008601 + +#define REG_A6XX_GRAS_PERFCTR_TSE_SEL_0 0x00008610 + +#define REG_A6XX_GRAS_PERFCTR_TSE_SEL_1 0x00008611 + +#define REG_A6XX_GRAS_PERFCTR_TSE_SEL_2 0x00008612 + +#define REG_A6XX_GRAS_PERFCTR_TSE_SEL_3 0x00008613 + +#define REG_A6XX_GRAS_PERFCTR_RAS_SEL_0 0x00008614 + +#define REG_A6XX_GRAS_PERFCTR_RAS_SEL_1 0x00008615 + +#define REG_A6XX_GRAS_PERFCTR_RAS_SEL_2 0x00008616 + +#define REG_A6XX_GRAS_PERFCTR_RAS_SEL_3 0x00008617 + +#define REG_A6XX_GRAS_PERFCTR_LRZ_SEL_0 0x00008618 + +#define REG_A6XX_GRAS_PERFCTR_LRZ_SEL_1 0x00008619 + +#define REG_A6XX_GRAS_PERFCTR_LRZ_SEL_2 0x0000861a + +#define REG_A6XX_GRAS_PERFCTR_LRZ_SEL_3 0x0000861b + +#define REG_A6XX_RB_ADDR_MODE_CNTL 0x00008e05 + +#define REG_A6XX_RB_NC_MODE_CNTL 0x00008e08 + +#define REG_A6XX_RB_PERFCTR_RB_SEL_0 0x00008e10 + +#define REG_A6XX_RB_PERFCTR_RB_SEL_1 0x00008e11 + +#define REG_A6XX_RB_PERFCTR_RB_SEL_2 0x00008e12 + +#define REG_A6XX_RB_PERFCTR_RB_SEL_3 0x00008e13 + +#define REG_A6XX_RB_PERFCTR_RB_SEL_4 0x00008e14 + +#define REG_A6XX_RB_PERFCTR_RB_SEL_5 0x00008e15 + +#define REG_A6XX_RB_PERFCTR_RB_SEL_6 0x00008e16 + +#define REG_A6XX_RB_PERFCTR_RB_SEL_7 0x00008e17 + +#define REG_A6XX_RB_PERFCTR_CCU_SEL_0 0x00008e18 + +#define REG_A6XX_RB_PERFCTR_CCU_SEL_1 0x00008e19 + +#define REG_A6XX_RB_PERFCTR_CCU_SEL_2 0x00008e1a + +#define REG_A6XX_RB_PERFCTR_CCU_SEL_3 0x00008e1b + +#define REG_A6XX_RB_PERFCTR_CCU_SEL_4 0x00008e1c + +#define REG_A6XX_RB_PERFCTR_CMP_SEL_0 0x00008e2c + +#define REG_A6XX_RB_PERFCTR_CMP_SEL_1 0x00008e2d + +#define REG_A6XX_RB_PERFCTR_CMP_SEL_2 0x00008e2e + +#define REG_A6XX_RB_PERFCTR_CMP_SEL_3 0x00008e2f + +#define REG_A6XX_RB_RB_SUB_BLOCK_SEL_CNTL_CD 0x00008e3d + +#define REG_A6XX_RB_CONTEXT_SWITCH_GMEM_SAVE_RESTORE 0x00008e50 + +#define REG_A6XX_PC_DBG_ECO_CNTL 0x00009e00 + +#define REG_A6XX_PC_ADDR_MODE_CNTL 0x00009e01 + +#define REG_A6XX_PC_PERFCTR_PC_SEL_0 0x00009e34 + +#define REG_A6XX_PC_PERFCTR_PC_SEL_1 0x00009e35 + +#define REG_A6XX_PC_PERFCTR_PC_SEL_2 0x00009e36 + +#define REG_A6XX_PC_PERFCTR_PC_SEL_3 0x00009e37 + +#define REG_A6XX_PC_PERFCTR_PC_SEL_4 0x00009e38 + +#define REG_A6XX_PC_PERFCTR_PC_SEL_5 0x00009e39 + +#define REG_A6XX_PC_PERFCTR_PC_SEL_6 0x00009e3a + +#define REG_A6XX_PC_PERFCTR_PC_SEL_7 0x00009e3b + +#define REG_A6XX_HLSQ_ADDR_MODE_CNTL 0x0000be05 + +#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_0 0x0000be10 + +#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_1 0x0000be11 + +#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_2 0x0000be12 + +#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_3 0x0000be13 + +#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_4 0x0000be14 + +#define REG_A6XX_HLSQ_PERFCTR_HLSQ_SEL_5 0x0000be15 + +#define REG_A6XX_HLSQ_DBG_AHB_READ_APERTURE 0x0000c800 + +#define REG_A6XX_HLSQ_DBG_READ_SEL 0x0000d000 + +#define REG_A6XX_VFD_ADDR_MODE_CNTL 0x0000a601 + +#define REG_A6XX_VFD_PERFCTR_VFD_SEL_0 0x0000a610 + +#define REG_A6XX_VFD_PERFCTR_VFD_SEL_1 0x0000a611 + +#define REG_A6XX_VFD_PERFCTR_VFD_SEL_2 0x0000a612 + +#define REG_A6XX_VFD_PERFCTR_VFD_SEL_3 0x0000a613 + +#define REG_A6XX_VFD_PERFCTR_VFD_SEL_4 0x0000a614 + +#define REG_A6XX_VFD_PERFCTR_VFD_SEL_5 0x0000a615 + +#define REG_A6XX_VFD_PERFCTR_VFD_SEL_6 0x0000a616 + +#define REG_A6XX_VFD_PERFCTR_VFD_SEL_7 0x0000a617 + +#define REG_A6XX_VPC_ADDR_MODE_CNTL 0x00009601 + +#define REG_A6XX_VPC_PERFCTR_VPC_SEL_0 0x00009604 + +#define REG_A6XX_VPC_PERFCTR_VPC_SEL_1 0x00009605 + +#define REG_A6XX_VPC_PERFCTR_VPC_SEL_2 0x00009606 + +#define REG_A6XX_VPC_PERFCTR_VPC_SEL_3 0x00009607 + +#define REG_A6XX_VPC_PERFCTR_VPC_SEL_4 0x00009608 + +#define REG_A6XX_VPC_PERFCTR_VPC_SEL_5 0x00009609 + +#define REG_A6XX_UCHE_ADDR_MODE_CNTL 0x00000e00 + +#define REG_A6XX_UCHE_MODE_CNTL 0x00000e01 + +#define REG_A6XX_UCHE_WRITE_RANGE_MAX_LO 0x00000e05 + +#define REG_A6XX_UCHE_WRITE_RANGE_MAX_HI 0x00000e06 + +#define REG_A6XX_UCHE_WRITE_THRU_BASE_LO 0x00000e07 + +#define REG_A6XX_UCHE_WRITE_THRU_BASE_HI 0x00000e08 + +#define REG_A6XX_UCHE_TRAP_BASE_LO 0x00000e09 + +#define REG_A6XX_UCHE_TRAP_BASE_HI 0x00000e0a + +#define REG_A6XX_UCHE_GMEM_RANGE_MIN_LO 0x00000e0b + +#define REG_A6XX_UCHE_GMEM_RANGE_MIN_HI 0x00000e0c + +#define REG_A6XX_UCHE_GMEM_RANGE_MAX_LO 0x00000e0d + +#define REG_A6XX_UCHE_GMEM_RANGE_MAX_HI 0x00000e0e + +#define REG_A6XX_UCHE_CACHE_WAYS 0x00000e17 + +#define REG_A6XX_UCHE_FILTER_CNTL 0x00000e18 + +#define REG_A6XX_UCHE_CLIENT_PF 0x00000e19 +#define A6XX_UCHE_CLIENT_PF_PERFSEL__MASK 0x000000ff +#define A6XX_UCHE_CLIENT_PF_PERFSEL__SHIFT 0 +static inline uint32_t A6XX_UCHE_CLIENT_PF_PERFSEL(uint32_t val) +{ + return ((val) << A6XX_UCHE_CLIENT_PF_PERFSEL__SHIFT) & A6XX_UCHE_CLIENT_PF_PERFSEL__MASK; +} + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_0 0x00000e1c + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_1 0x00000e1d + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_2 0x00000e1e + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_3 0x00000e1f + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_4 0x00000e20 + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_5 0x00000e21 + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_6 0x00000e22 + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_7 0x00000e23 + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_8 0x00000e24 + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_9 0x00000e25 + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_10 0x00000e26 + +#define REG_A6XX_UCHE_PERFCTR_UCHE_SEL_11 0x00000e27 + +#define REG_A6XX_SP_ADDR_MODE_CNTL 0x0000ae01 + +#define REG_A6XX_SP_NC_MODE_CNTL 0x0000ae02 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_0 0x0000ae10 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_1 0x0000ae11 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_2 0x0000ae12 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_3 0x0000ae13 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_4 0x0000ae14 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_5 0x0000ae15 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_6 0x0000ae16 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_7 0x0000ae17 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_8 0x0000ae18 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_9 0x0000ae19 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_10 0x0000ae1a + +#define REG_A6XX_SP_PERFCTR_SP_SEL_11 0x0000ae1b + +#define REG_A6XX_SP_PERFCTR_SP_SEL_12 0x0000ae1c + +#define REG_A6XX_SP_PERFCTR_SP_SEL_13 0x0000ae1d + +#define REG_A6XX_SP_PERFCTR_SP_SEL_14 0x0000ae1e + +#define REG_A6XX_SP_PERFCTR_SP_SEL_15 0x0000ae1f + +#define REG_A6XX_SP_PERFCTR_SP_SEL_16 0x0000ae20 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_17 0x0000ae21 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_18 0x0000ae22 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_19 0x0000ae23 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_20 0x0000ae24 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_21 0x0000ae25 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_22 0x0000ae26 + +#define REG_A6XX_SP_PERFCTR_SP_SEL_23 0x0000ae27 + +#define REG_A6XX_TPL1_ADDR_MODE_CNTL 0x0000b601 + +#define REG_A6XX_TPL1_NC_MODE_CNTL 0x0000b604 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_0 0x0000b610 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_1 0x0000b611 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_2 0x0000b612 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_3 0x0000b613 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_4 0x0000b614 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_5 0x0000b615 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_6 0x0000b616 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_7 0x0000b617 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_8 0x0000b618 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_9 0x0000b619 + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_10 0x0000b61a + +#define REG_A6XX_TPL1_PERFCTR_TP_SEL_11 0x0000b61b + +#define REG_A6XX_VBIF_VERSION 0x00003000 + +#define REG_A6XX_VBIF_GATE_OFF_WRREQ_EN 0x0000302a + +#define REG_A6XX_VBIF_XIN_HALT_CTRL0 0x00003080 + +#define REG_A6XX_VBIF_XIN_HALT_CTRL1 0x00003081 + +#define REG_A6XX_VBIF_PERF_CNT_SEL0 0x000030d0 + +#define REG_A6XX_VBIF_PERF_CNT_SEL1 0x000030d1 + +#define REG_A6XX_VBIF_PERF_CNT_SEL2 0x000030d2 + +#define REG_A6XX_VBIF_PERF_CNT_SEL3 0x000030d3 + +#define REG_A6XX_VBIF_PERF_CNT_LOW0 0x000030d8 + +#define REG_A6XX_VBIF_PERF_CNT_LOW1 0x000030d9 + +#define REG_A6XX_VBIF_PERF_CNT_LOW2 0x000030da + +#define REG_A6XX_VBIF_PERF_CNT_LOW3 0x000030db + +#define REG_A6XX_VBIF_PERF_CNT_HIGH0 0x000030e0 + +#define REG_A6XX_VBIF_PERF_CNT_HIGH1 0x000030e1 + +#define REG_A6XX_VBIF_PERF_CNT_HIGH2 0x000030e2 + +#define REG_A6XX_VBIF_PERF_CNT_HIGH3 0x000030e3 + +#define REG_A6XX_VBIF_PERF_PWR_CNT_EN0 0x00003100 + +#define REG_A6XX_VBIF_PERF_PWR_CNT_EN1 0x00003101 + +#define REG_A6XX_VBIF_PERF_PWR_CNT_EN2 0x00003102 + +#define REG_A6XX_VBIF_PERF_PWR_CNT_LOW0 0x00003110 + +#define REG_A6XX_VBIF_PERF_PWR_CNT_LOW1 0x00003111 + +#define REG_A6XX_VBIF_PERF_PWR_CNT_LOW2 0x00003112 + +#define REG_A6XX_VBIF_PERF_PWR_CNT_HIGH0 0x00003118 + +#define REG_A6XX_VBIF_PERF_PWR_CNT_HIGH1 0x00003119 + +#define REG_A6XX_VBIF_PERF_PWR_CNT_HIGH2 0x0000311a + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_A 0x00018400 + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_B 0x00018401 + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_C 0x00018402 + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_SEL_D 0x00018403 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK 0x000000ff +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT 0 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_INDEX__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK 0x0000ff00 +#define A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT 8 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_SEL_D_PING_BLK_SEL__MASK; +} + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_CNTLT 0x00018404 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK 0x0000003f +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT 0 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_TRACEEN__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK 0x00007000 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT 12 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_GRANU__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK 0xf0000000 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT 28 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLT_SEGT__MASK; +} + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_CNTLM 0x00018405 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK 0x0f000000 +#define A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT 24 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_CNTLM_ENABLE__MASK; +} + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_0 0x00018408 + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_1 0x00018409 + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_2 0x0001840a + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_IVTL_3 0x0001840b + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_0 0x0001840c + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_1 0x0001840d + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_2 0x0001840e + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_MASKL_3 0x0001840f + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0 0x00018410 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK 0x0000000f +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT 0 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL0__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK 0x000000f0 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT 4 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL1__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK 0x00000f00 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT 8 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL2__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK 0x0000f000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT 12 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL3__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK 0x000f0000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT 16 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL4__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK 0x00f00000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT 20 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL5__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK 0x0f000000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT 24 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL6__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK 0xf0000000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT 28 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_0_BYTEL7__MASK; +} + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1 0x00018411 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK 0x0000000f +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT 0 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL8__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK 0x000000f0 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT 4 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL9__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK 0x00000f00 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT 8 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL10__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK 0x0000f000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT 12 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL11__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK 0x000f0000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT 16 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL12__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK 0x00f00000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT 20 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL13__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK 0x0f000000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT 24 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL14__MASK; +} +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK 0xf0000000 +#define A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT 28 +static inline uint32_t A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15(uint32_t val) +{ + return ((val) << A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__SHIFT) & A6XX_CX_DBGC_CFG_DBGBUS_BYTEL_1_BYTEL15__MASK; +} + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF1 0x0001842f + +#define REG_A6XX_CX_DBGC_CFG_DBGBUS_TRACE_BUF2 0x00018430 + +#define REG_A6XX_PDC_GPU_ENABLE_PDC 0x00021140 + +#define REG_A6XX_PDC_GPU_SEQ_START_ADDR 0x00021148 + +#define REG_A6XX_PDC_GPU_TCS0_CONTROL 0x00021540 + +#define REG_A6XX_PDC_GPU_TCS0_CMD_ENABLE_BANK 0x00021541 + +#define REG_A6XX_PDC_GPU_TCS0_CMD_WAIT_FOR_CMPL_BANK 0x00021542 + +#define REG_A6XX_PDC_GPU_TCS0_CMD0_MSGID 0x00021543 + +#define REG_A6XX_PDC_GPU_TCS0_CMD0_ADDR 0x00021544 + +#define REG_A6XX_PDC_GPU_TCS0_CMD0_DATA 0x00021545 + +#define REG_A6XX_PDC_GPU_TCS1_CONTROL 0x00021572 + +#define REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK 0x00021573 + +#define REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK 0x00021574 + +#define REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID 0x00021575 + +#define REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR 0x00021576 + +#define REG_A6XX_PDC_GPU_TCS1_CMD0_DATA 0x00021577 + +#define REG_A6XX_PDC_GPU_TCS2_CONTROL 0x000215a4 + +#define REG_A6XX_PDC_GPU_TCS2_CMD_ENABLE_BANK 0x000215a5 + +#define REG_A6XX_PDC_GPU_TCS2_CMD_WAIT_FOR_CMPL_BANK 0x000215a6 + +#define REG_A6XX_PDC_GPU_TCS2_CMD0_MSGID 0x000215a7 + +#define REG_A6XX_PDC_GPU_TCS2_CMD0_ADDR 0x000215a8 + +#define REG_A6XX_PDC_GPU_TCS2_CMD0_DATA 0x000215a9 + +#define REG_A6XX_PDC_GPU_TCS3_CONTROL 0x000215d6 + +#define REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK 0x000215d7 + +#define REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK 0x000215d8 + +#define REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID 0x000215d9 + +#define REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR 0x000215da + +#define REG_A6XX_PDC_GPU_TCS3_CMD0_DATA 0x000215db + +#define REG_A6XX_PDC_GPU_SEQ_MEM_0 0x000a0000 + +#define REG_A6XX_X1_WINDOW_OFFSET 0x000088d4 +#define A6XX_X1_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_X1_WINDOW_OFFSET_X__MASK 0x00007fff +#define A6XX_X1_WINDOW_OFFSET_X__SHIFT 0 +static inline uint32_t A6XX_X1_WINDOW_OFFSET_X(uint32_t val) +{ + return ((val) << A6XX_X1_WINDOW_OFFSET_X__SHIFT) & A6XX_X1_WINDOW_OFFSET_X__MASK; +} +#define A6XX_X1_WINDOW_OFFSET_Y__MASK 0x7fff0000 +#define A6XX_X1_WINDOW_OFFSET_Y__SHIFT 16 +static inline uint32_t A6XX_X1_WINDOW_OFFSET_Y(uint32_t val) +{ + return ((val) << A6XX_X1_WINDOW_OFFSET_Y__SHIFT) & A6XX_X1_WINDOW_OFFSET_Y__MASK; +} + +#define REG_A6XX_X2_WINDOW_OFFSET 0x0000b4d1 +#define A6XX_X2_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_X2_WINDOW_OFFSET_X__MASK 0x00007fff +#define A6XX_X2_WINDOW_OFFSET_X__SHIFT 0 +static inline uint32_t A6XX_X2_WINDOW_OFFSET_X(uint32_t val) +{ + return ((val) << A6XX_X2_WINDOW_OFFSET_X__SHIFT) & A6XX_X2_WINDOW_OFFSET_X__MASK; +} +#define A6XX_X2_WINDOW_OFFSET_Y__MASK 0x7fff0000 +#define A6XX_X2_WINDOW_OFFSET_Y__SHIFT 16 +static inline uint32_t A6XX_X2_WINDOW_OFFSET_Y(uint32_t val) +{ + return ((val) << A6XX_X2_WINDOW_OFFSET_Y__SHIFT) & A6XX_X2_WINDOW_OFFSET_Y__MASK; +} + +#define REG_A6XX_X3_WINDOW_OFFSET 0x0000b307 +#define A6XX_X3_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_X3_WINDOW_OFFSET_X__MASK 0x00007fff +#define A6XX_X3_WINDOW_OFFSET_X__SHIFT 0 +static inline uint32_t A6XX_X3_WINDOW_OFFSET_X(uint32_t val) +{ + return ((val) << A6XX_X3_WINDOW_OFFSET_X__SHIFT) & A6XX_X3_WINDOW_OFFSET_X__MASK; +} +#define A6XX_X3_WINDOW_OFFSET_Y__MASK 0x7fff0000 +#define A6XX_X3_WINDOW_OFFSET_Y__SHIFT 16 +static inline uint32_t A6XX_X3_WINDOW_OFFSET_Y(uint32_t val) +{ + return ((val) << A6XX_X3_WINDOW_OFFSET_Y__SHIFT) & A6XX_X3_WINDOW_OFFSET_Y__MASK; +} + +#define REG_A6XX_X1_BIN_SIZE 0x000080a1 +#define A6XX_X1_BIN_SIZE_WIDTH__MASK 0x000000ff +#define A6XX_X1_BIN_SIZE_WIDTH__SHIFT 0 +static inline uint32_t A6XX_X1_BIN_SIZE_WIDTH(uint32_t val) +{ + return ((val >> 5) << A6XX_X1_BIN_SIZE_WIDTH__SHIFT) & A6XX_X1_BIN_SIZE_WIDTH__MASK; +} +#define A6XX_X1_BIN_SIZE_HEIGHT__MASK 0x0001ff00 +#define A6XX_X1_BIN_SIZE_HEIGHT__SHIFT 8 +static inline uint32_t A6XX_X1_BIN_SIZE_HEIGHT(uint32_t val) +{ + return ((val >> 4) << A6XX_X1_BIN_SIZE_HEIGHT__SHIFT) & A6XX_X1_BIN_SIZE_HEIGHT__MASK; +} + +#define REG_A6XX_X2_BIN_SIZE 0x00008800 +#define A6XX_X2_BIN_SIZE_WIDTH__MASK 0x000000ff +#define A6XX_X2_BIN_SIZE_WIDTH__SHIFT 0 +static inline uint32_t A6XX_X2_BIN_SIZE_WIDTH(uint32_t val) +{ + return ((val >> 5) << A6XX_X2_BIN_SIZE_WIDTH__SHIFT) & A6XX_X2_BIN_SIZE_WIDTH__MASK; +} +#define A6XX_X2_BIN_SIZE_HEIGHT__MASK 0x0001ff00 +#define A6XX_X2_BIN_SIZE_HEIGHT__SHIFT 8 +static inline uint32_t A6XX_X2_BIN_SIZE_HEIGHT(uint32_t val) +{ + return ((val >> 4) << A6XX_X2_BIN_SIZE_HEIGHT__SHIFT) & A6XX_X2_BIN_SIZE_HEIGHT__MASK; +} + +#define REG_A6XX_X3_BIN_SIZE 0x000088d3 +#define A6XX_X3_BIN_SIZE_WIDTH__MASK 0x000000ff +#define A6XX_X3_BIN_SIZE_WIDTH__SHIFT 0 +static inline uint32_t A6XX_X3_BIN_SIZE_WIDTH(uint32_t val) +{ + return ((val >> 5) << A6XX_X3_BIN_SIZE_WIDTH__SHIFT) & A6XX_X3_BIN_SIZE_WIDTH__MASK; +} +#define A6XX_X3_BIN_SIZE_HEIGHT__MASK 0x0001ff00 +#define A6XX_X3_BIN_SIZE_HEIGHT__SHIFT 8 +static inline uint32_t A6XX_X3_BIN_SIZE_HEIGHT(uint32_t val) +{ + return ((val >> 4) << A6XX_X3_BIN_SIZE_HEIGHT__SHIFT) & A6XX_X3_BIN_SIZE_HEIGHT__MASK; +} + +#define REG_A6XX_VSC_BIN_SIZE 0x00000c02 +#define A6XX_VSC_BIN_SIZE_WIDTH__MASK 0x000000ff +#define A6XX_VSC_BIN_SIZE_WIDTH__SHIFT 0 +static inline uint32_t A6XX_VSC_BIN_SIZE_WIDTH(uint32_t val) +{ + return ((val >> 5) << A6XX_VSC_BIN_SIZE_WIDTH__SHIFT) & A6XX_VSC_BIN_SIZE_WIDTH__MASK; +} +#define A6XX_VSC_BIN_SIZE_HEIGHT__MASK 0x0001ff00 +#define A6XX_VSC_BIN_SIZE_HEIGHT__SHIFT 8 +static inline uint32_t A6XX_VSC_BIN_SIZE_HEIGHT(uint32_t val) +{ + return ((val >> 4) << A6XX_VSC_BIN_SIZE_HEIGHT__SHIFT) & A6XX_VSC_BIN_SIZE_HEIGHT__MASK; +} + +#define REG_A6XX_VSC_SIZE_ADDRESS_LO 0x00000c03 + +#define REG_A6XX_VSC_SIZE_ADDRESS_HI 0x00000c04 + +#define REG_A6XX_VSC_BIN_COUNT 0x00000c06 +#define A6XX_VSC_BIN_COUNT_NX__MASK 0x000007fe +#define A6XX_VSC_BIN_COUNT_NX__SHIFT 1 +static inline uint32_t A6XX_VSC_BIN_COUNT_NX(uint32_t val) +{ + return ((val) << A6XX_VSC_BIN_COUNT_NX__SHIFT) & A6XX_VSC_BIN_COUNT_NX__MASK; +} +#define A6XX_VSC_BIN_COUNT_NY__MASK 0x001ff800 +#define A6XX_VSC_BIN_COUNT_NY__SHIFT 11 +static inline uint32_t A6XX_VSC_BIN_COUNT_NY(uint32_t val) +{ + return ((val) << A6XX_VSC_BIN_COUNT_NY__SHIFT) & A6XX_VSC_BIN_COUNT_NY__MASK; +} + +static inline uint32_t REG_A6XX_VSC_PIPE_CONFIG(uint32_t i0) { return 0x00000c10 + 0x1*i0; } + +static inline uint32_t REG_A6XX_VSC_PIPE_CONFIG_REG(uint32_t i0) { return 0x00000c10 + 0x1*i0; } +#define A6XX_VSC_PIPE_CONFIG_REG_X__MASK 0x000003ff +#define A6XX_VSC_PIPE_CONFIG_REG_X__SHIFT 0 +static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_X(uint32_t val) +{ + return ((val) << A6XX_VSC_PIPE_CONFIG_REG_X__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_X__MASK; +} +#define A6XX_VSC_PIPE_CONFIG_REG_Y__MASK 0x000ffc00 +#define A6XX_VSC_PIPE_CONFIG_REG_Y__SHIFT 10 +static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_Y(uint32_t val) +{ + return ((val) << A6XX_VSC_PIPE_CONFIG_REG_Y__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_Y__MASK; +} +#define A6XX_VSC_PIPE_CONFIG_REG_W__MASK 0x03f00000 +#define A6XX_VSC_PIPE_CONFIG_REG_W__SHIFT 20 +static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_W(uint32_t val) +{ + return ((val) << A6XX_VSC_PIPE_CONFIG_REG_W__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_W__MASK; +} +#define A6XX_VSC_PIPE_CONFIG_REG_H__MASK 0xfc000000 +#define A6XX_VSC_PIPE_CONFIG_REG_H__SHIFT 26 +static inline uint32_t A6XX_VSC_PIPE_CONFIG_REG_H(uint32_t val) +{ + return ((val) << A6XX_VSC_PIPE_CONFIG_REG_H__SHIFT) & A6XX_VSC_PIPE_CONFIG_REG_H__MASK; +} + +#define REG_A6XX_VSC_XXX_ADDRESS_LO 0x00000c30 + +#define REG_A6XX_VSC_XXX_ADDRESS_HI 0x00000c31 + +#define REG_A6XX_VSC_XXX_PITCH 0x00000c32 + +#define REG_A6XX_VSC_PIPE_DATA_ADDRESS_LO 0x00000c34 + +#define REG_A6XX_VSC_PIPE_DATA_ADDRESS_HI 0x00000c35 + +#define REG_A6XX_VSC_PIPE_DATA_PITCH 0x00000c36 + +static inline uint32_t REG_A6XX_VSC_SIZE(uint32_t i0) { return 0x00000c78 + 0x1*i0; } + +static inline uint32_t REG_A6XX_VSC_SIZE_REG(uint32_t i0) { return 0x00000c78 + 0x1*i0; } + +#define REG_A6XX_UCHE_UNKNOWN_0E12 0x00000e12 + +#define REG_A6XX_GRAS_UNKNOWN_8001 0x00008001 + +#define REG_A6XX_GRAS_UNKNOWN_8004 0x00008004 + +#define REG_A6XX_GRAS_CNTL 0x00008005 +#define A6XX_GRAS_CNTL_VARYING 0x00000001 +#define A6XX_GRAS_CNTL_XCOORD 0x00000040 +#define A6XX_GRAS_CNTL_YCOORD 0x00000080 +#define A6XX_GRAS_CNTL_ZCOORD 0x00000100 +#define A6XX_GRAS_CNTL_WCOORD 0x00000200 + +#define REG_A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ 0x00008006 +#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK 0x000003ff +#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT 0 +static inline uint32_t A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ(uint32_t val) +{ + return ((val) << A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__SHIFT) & A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_HORZ__MASK; +} +#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK 0x000ffc00 +#define A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT 10 +static inline uint32_t A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT(uint32_t val) +{ + return ((val) << A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__SHIFT) & A6XX_GRAS_CL_GUARDBAND_CLIP_ADJ_VERT__MASK; +} + +#define REG_A6XX_GRAS_CL_VPORT_XOFFSET_0 0x00008010 +#define A6XX_GRAS_CL_VPORT_XOFFSET_0__MASK 0xffffffff +#define A6XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT 0 +static inline uint32_t A6XX_GRAS_CL_VPORT_XOFFSET_0(float val) +{ + return ((fui(val)) << A6XX_GRAS_CL_VPORT_XOFFSET_0__SHIFT) & A6XX_GRAS_CL_VPORT_XOFFSET_0__MASK; +} + +#define REG_A6XX_GRAS_CL_VPORT_XSCALE_0 0x00008011 +#define A6XX_GRAS_CL_VPORT_XSCALE_0__MASK 0xffffffff +#define A6XX_GRAS_CL_VPORT_XSCALE_0__SHIFT 0 +static inline uint32_t A6XX_GRAS_CL_VPORT_XSCALE_0(float val) +{ + return ((fui(val)) << A6XX_GRAS_CL_VPORT_XSCALE_0__SHIFT) & A6XX_GRAS_CL_VPORT_XSCALE_0__MASK; +} + +#define REG_A6XX_GRAS_CL_VPORT_YOFFSET_0 0x00008012 +#define A6XX_GRAS_CL_VPORT_YOFFSET_0__MASK 0xffffffff +#define A6XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT 0 +static inline uint32_t A6XX_GRAS_CL_VPORT_YOFFSET_0(float val) +{ + return ((fui(val)) << A6XX_GRAS_CL_VPORT_YOFFSET_0__SHIFT) & A6XX_GRAS_CL_VPORT_YOFFSET_0__MASK; +} + +#define REG_A6XX_GRAS_CL_VPORT_YSCALE_0 0x00008013 +#define A6XX_GRAS_CL_VPORT_YSCALE_0__MASK 0xffffffff +#define A6XX_GRAS_CL_VPORT_YSCALE_0__SHIFT 0 +static inline uint32_t A6XX_GRAS_CL_VPORT_YSCALE_0(float val) +{ + return ((fui(val)) << A6XX_GRAS_CL_VPORT_YSCALE_0__SHIFT) & A6XX_GRAS_CL_VPORT_YSCALE_0__MASK; +} + +#define REG_A6XX_GRAS_CL_VPORT_ZOFFSET_0 0x00008014 +#define A6XX_GRAS_CL_VPORT_ZOFFSET_0__MASK 0xffffffff +#define A6XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT 0 +static inline uint32_t A6XX_GRAS_CL_VPORT_ZOFFSET_0(float val) +{ + return ((fui(val)) << A6XX_GRAS_CL_VPORT_ZOFFSET_0__SHIFT) & A6XX_GRAS_CL_VPORT_ZOFFSET_0__MASK; +} + +#define REG_A6XX_GRAS_CL_VPORT_ZSCALE_0 0x00008015 +#define A6XX_GRAS_CL_VPORT_ZSCALE_0__MASK 0xffffffff +#define A6XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT 0 +static inline uint32_t A6XX_GRAS_CL_VPORT_ZSCALE_0(float val) +{ + return ((fui(val)) << A6XX_GRAS_CL_VPORT_ZSCALE_0__SHIFT) & A6XX_GRAS_CL_VPORT_ZSCALE_0__MASK; +} + +#define REG_A6XX_GRAS_SU_CNTL 0x00008090 +#define A6XX_GRAS_SU_CNTL_CULL_FRONT 0x00000001 +#define A6XX_GRAS_SU_CNTL_CULL_BACK 0x00000002 +#define A6XX_GRAS_SU_CNTL_FRONT_CW 0x00000004 +#define A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK 0x000007f8 +#define A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT 3 +static inline uint32_t A6XX_GRAS_SU_CNTL_LINEHALFWIDTH(float val) +{ + return ((((int32_t)(val * 4.0))) << A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__SHIFT) & A6XX_GRAS_SU_CNTL_LINEHALFWIDTH__MASK; +} +#define A6XX_GRAS_SU_CNTL_POLY_OFFSET 0x00000800 +#define A6XX_GRAS_SU_CNTL_MSAA_ENABLE 0x00002000 + +#define REG_A6XX_GRAS_SU_POINT_MINMAX 0x00008091 +#define A6XX_GRAS_SU_POINT_MINMAX_MIN__MASK 0x0000ffff +#define A6XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT 0 +static inline uint32_t A6XX_GRAS_SU_POINT_MINMAX_MIN(float val) +{ + return ((((uint32_t)(val * 16.0))) << A6XX_GRAS_SU_POINT_MINMAX_MIN__SHIFT) & A6XX_GRAS_SU_POINT_MINMAX_MIN__MASK; +} +#define A6XX_GRAS_SU_POINT_MINMAX_MAX__MASK 0xffff0000 +#define A6XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT 16 +static inline uint32_t A6XX_GRAS_SU_POINT_MINMAX_MAX(float val) +{ + return ((((uint32_t)(val * 16.0))) << A6XX_GRAS_SU_POINT_MINMAX_MAX__SHIFT) & A6XX_GRAS_SU_POINT_MINMAX_MAX__MASK; +} + +#define REG_A6XX_GRAS_SU_POINT_SIZE 0x00008092 +#define A6XX_GRAS_SU_POINT_SIZE__MASK 0xffffffff +#define A6XX_GRAS_SU_POINT_SIZE__SHIFT 0 +static inline uint32_t A6XX_GRAS_SU_POINT_SIZE(float val) +{ + return ((((int32_t)(val * 16.0))) << A6XX_GRAS_SU_POINT_SIZE__SHIFT) & A6XX_GRAS_SU_POINT_SIZE__MASK; +} + +#define REG_A6XX_GRAS_SU_POLY_OFFSET_SCALE 0x00008095 +#define A6XX_GRAS_SU_POLY_OFFSET_SCALE__MASK 0xffffffff +#define A6XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT 0 +static inline uint32_t A6XX_GRAS_SU_POLY_OFFSET_SCALE(float val) +{ + return ((fui(val)) << A6XX_GRAS_SU_POLY_OFFSET_SCALE__SHIFT) & A6XX_GRAS_SU_POLY_OFFSET_SCALE__MASK; +} + +#define REG_A6XX_GRAS_SU_POLY_OFFSET_OFFSET 0x00008096 +#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK 0xffffffff +#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT 0 +static inline uint32_t A6XX_GRAS_SU_POLY_OFFSET_OFFSET(float val) +{ + return ((fui(val)) << A6XX_GRAS_SU_POLY_OFFSET_OFFSET__SHIFT) & A6XX_GRAS_SU_POLY_OFFSET_OFFSET__MASK; +} + +#define REG_A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP 0x00008097 +#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK 0xffffffff +#define A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT 0 +static inline uint32_t A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP(float val) +{ + return ((fui(val)) << A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__SHIFT) & A6XX_GRAS_SU_POLY_OFFSET_OFFSET_CLAMP__MASK; +} + +#define REG_A6XX_GRAS_SU_DEPTH_BUFFER_INFO 0x00008098 +#define A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK 0x00000007 +#define A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT 0 +static inline uint32_t A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a6xx_depth_format val) +{ + return ((val) << A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A6XX_GRAS_SU_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK; +} + +#define REG_A6XX_GRAS_UNKNOWN_8099 0x00008099 + +#define REG_A6XX_GRAS_UNKNOWN_809B 0x0000809b + +#define REG_A6XX_GRAS_RAS_MSAA_CNTL 0x000080a2 +#define A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__MASK 0x00000003 +#define A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__SHIFT 0 +static inline uint32_t A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val) +{ + return ((val) << A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_GRAS_RAS_MSAA_CNTL_SAMPLES__MASK; +} + +#define REG_A6XX_GRAS_DEST_MSAA_CNTL 0x000080a3 +#define A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__MASK 0x00000003 +#define A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__SHIFT 0 +static inline uint32_t A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val) +{ + return ((val) << A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_GRAS_DEST_MSAA_CNTL_SAMPLES__MASK; +} +#define A6XX_GRAS_DEST_MSAA_CNTL_MSAA_DISABLE 0x00000004 + +#define REG_A6XX_GRAS_UNKNOWN_80A4 0x000080a4 + +#define REG_A6XX_GRAS_UNKNOWN_80A5 0x000080a5 + +#define REG_A6XX_GRAS_UNKNOWN_80A6 0x000080a6 + +#define REG_A6XX_GRAS_UNKNOWN_80AF 0x000080af + +#define REG_A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0 0x000080b0 +#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__MASK 0x00007fff +#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_X__MASK; +} +#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__MASK 0x7fff0000 +#define A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_TL_0_Y__MASK; +} + +#define REG_A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0 0x000080b1 +#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__MASK 0x00007fff +#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_X__MASK; +} +#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__MASK 0x7fff0000 +#define A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__SHIFT) & A6XX_GRAS_SC_SCREEN_SCISSOR_BR_0_Y__MASK; +} + +#define REG_A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0 0x000080d0 +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__MASK 0x00007fff +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_X__MASK; +} +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__MASK 0x7fff0000 +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_TL_0_Y__MASK; +} + +#define REG_A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0 0x000080d1 +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__MASK 0x00007fff +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_X__MASK; +} +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__MASK 0x7fff0000 +#define A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__SHIFT) & A6XX_GRAS_SC_VIEWPORT_SCISSOR_BR_0_Y__MASK; +} + +#define REG_A6XX_GRAS_SC_WINDOW_SCISSOR_TL 0x000080f0 +#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK 0x00007fff +#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_TL_X__MASK; +} +#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK 0x7fff0000 +#define A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_TL_Y__MASK; +} + +#define REG_A6XX_GRAS_SC_WINDOW_SCISSOR_BR 0x000080f1 +#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK 0x00007fff +#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_BR_X__MASK; +} +#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK 0x7fff0000 +#define A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__SHIFT) & A6XX_GRAS_SC_WINDOW_SCISSOR_BR_Y__MASK; +} + +#define REG_A6XX_GRAS_LRZ_CNTL 0x00008100 +#define A6XX_GRAS_LRZ_CNTL_ENABLE 0x00000001 +#define A6XX_GRAS_LRZ_CNTL_LRZ_WRITE 0x00000002 +#define A6XX_GRAS_LRZ_CNTL_GREATER 0x00000004 + +#define REG_A6XX_GRAS_2D_BLIT_INFO 0x00008102 +#define A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT__MASK 0x000000ff +#define A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT__SHIFT 0 +static inline uint32_t A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT(enum a6xx_color_fmt val) +{ + return ((val) << A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT__SHIFT) & A6XX_GRAS_2D_BLIT_INFO_COLOR_FORMAT__MASK; +} + +#define REG_A6XX_GRAS_LRZ_BUFFER_BASE_LO 0x00008103 + +#define REG_A6XX_GRAS_LRZ_BUFFER_BASE_HI 0x00008104 + +#define REG_A6XX_GRAS_LRZ_BUFFER_PITCH 0x00008105 +#define A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__MASK 0x000007ff +#define A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__SHIFT 0 +static inline uint32_t A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH(uint32_t val) +{ + return ((val >> 5) << A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__SHIFT) & A6XX_GRAS_LRZ_BUFFER_PITCH_PITCH__MASK; +} +#define A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__MASK 0x003ff800 +#define A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__SHIFT 11 +static inline uint32_t A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH(uint32_t val) +{ + return ((val >> 5) << A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__SHIFT) & A6XX_GRAS_LRZ_BUFFER_PITCH_ARRAY_PITCH__MASK; +} + +#define REG_A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_LO 0x00008106 + +#define REG_A6XX_GRAS_LRZ_FAST_CLEAR_BUFFER_BASE_HI 0x00008107 + +#define REG_A6XX_GRAS_2D_BLIT_CNTL 0x00008400 + +#define REG_A6XX_GRAS_2D_SRC_TL_X 0x00008401 +#define A6XX_GRAS_2D_SRC_TL_X_X__MASK 0x00ffff00 +#define A6XX_GRAS_2D_SRC_TL_X_X__SHIFT 8 +static inline uint32_t A6XX_GRAS_2D_SRC_TL_X_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_2D_SRC_TL_X_X__SHIFT) & A6XX_GRAS_2D_SRC_TL_X_X__MASK; +} + +#define REG_A6XX_GRAS_2D_SRC_BR_X 0x00008402 +#define A6XX_GRAS_2D_SRC_BR_X_X__MASK 0x00ffff00 +#define A6XX_GRAS_2D_SRC_BR_X_X__SHIFT 8 +static inline uint32_t A6XX_GRAS_2D_SRC_BR_X_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_2D_SRC_BR_X_X__SHIFT) & A6XX_GRAS_2D_SRC_BR_X_X__MASK; +} + +#define REG_A6XX_GRAS_2D_SRC_TL_Y 0x00008403 +#define A6XX_GRAS_2D_SRC_TL_Y_Y__MASK 0x00ffff00 +#define A6XX_GRAS_2D_SRC_TL_Y_Y__SHIFT 8 +static inline uint32_t A6XX_GRAS_2D_SRC_TL_Y_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_2D_SRC_TL_Y_Y__SHIFT) & A6XX_GRAS_2D_SRC_TL_Y_Y__MASK; +} + +#define REG_A6XX_GRAS_2D_SRC_BR_Y 0x00008404 +#define A6XX_GRAS_2D_SRC_BR_Y_Y__MASK 0x00ffff00 +#define A6XX_GRAS_2D_SRC_BR_Y_Y__SHIFT 8 +static inline uint32_t A6XX_GRAS_2D_SRC_BR_Y_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_2D_SRC_BR_Y_Y__SHIFT) & A6XX_GRAS_2D_SRC_BR_Y_Y__MASK; +} + +#define REG_A6XX_GRAS_2D_DST_TL 0x00008405 +#define A6XX_GRAS_2D_DST_TL_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_2D_DST_TL_X__MASK 0x00007fff +#define A6XX_GRAS_2D_DST_TL_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_2D_DST_TL_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_2D_DST_TL_X__SHIFT) & A6XX_GRAS_2D_DST_TL_X__MASK; +} +#define A6XX_GRAS_2D_DST_TL_Y__MASK 0x7fff0000 +#define A6XX_GRAS_2D_DST_TL_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_2D_DST_TL_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_2D_DST_TL_Y__SHIFT) & A6XX_GRAS_2D_DST_TL_Y__MASK; +} + +#define REG_A6XX_GRAS_2D_DST_BR 0x00008406 +#define A6XX_GRAS_2D_DST_BR_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_2D_DST_BR_X__MASK 0x00007fff +#define A6XX_GRAS_2D_DST_BR_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_2D_DST_BR_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_2D_DST_BR_X__SHIFT) & A6XX_GRAS_2D_DST_BR_X__MASK; +} +#define A6XX_GRAS_2D_DST_BR_Y__MASK 0x7fff0000 +#define A6XX_GRAS_2D_DST_BR_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_2D_DST_BR_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_2D_DST_BR_Y__SHIFT) & A6XX_GRAS_2D_DST_BR_Y__MASK; +} + +#define REG_A6XX_GRAS_RESOLVE_CNTL_1 0x0000840a +#define A6XX_GRAS_RESOLVE_CNTL_1_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_RESOLVE_CNTL_1_X__MASK 0x00007fff +#define A6XX_GRAS_RESOLVE_CNTL_1_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_RESOLVE_CNTL_1_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_RESOLVE_CNTL_1_X__SHIFT) & A6XX_GRAS_RESOLVE_CNTL_1_X__MASK; +} +#define A6XX_GRAS_RESOLVE_CNTL_1_Y__MASK 0x7fff0000 +#define A6XX_GRAS_RESOLVE_CNTL_1_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_RESOLVE_CNTL_1_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_RESOLVE_CNTL_1_Y__SHIFT) & A6XX_GRAS_RESOLVE_CNTL_1_Y__MASK; +} + +#define REG_A6XX_GRAS_RESOLVE_CNTL_2 0x0000840b +#define A6XX_GRAS_RESOLVE_CNTL_2_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_GRAS_RESOLVE_CNTL_2_X__MASK 0x00007fff +#define A6XX_GRAS_RESOLVE_CNTL_2_X__SHIFT 0 +static inline uint32_t A6XX_GRAS_RESOLVE_CNTL_2_X(uint32_t val) +{ + return ((val) << A6XX_GRAS_RESOLVE_CNTL_2_X__SHIFT) & A6XX_GRAS_RESOLVE_CNTL_2_X__MASK; +} +#define A6XX_GRAS_RESOLVE_CNTL_2_Y__MASK 0x7fff0000 +#define A6XX_GRAS_RESOLVE_CNTL_2_Y__SHIFT 16 +static inline uint32_t A6XX_GRAS_RESOLVE_CNTL_2_Y(uint32_t val) +{ + return ((val) << A6XX_GRAS_RESOLVE_CNTL_2_Y__SHIFT) & A6XX_GRAS_RESOLVE_CNTL_2_Y__MASK; +} + +#define REG_A6XX_GRAS_UNKNOWN_8600 0x00008600 + +#define REG_A6XX_RB_RAS_MSAA_CNTL 0x00008802 +#define A6XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK 0x00000003 +#define A6XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT 0 +static inline uint32_t A6XX_RB_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val) +{ + return ((val) << A6XX_RB_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_RB_RAS_MSAA_CNTL_SAMPLES__MASK; +} + +#define REG_A6XX_RB_DEST_MSAA_CNTL 0x00008803 +#define A6XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK 0x00000003 +#define A6XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT 0 +static inline uint32_t A6XX_RB_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val) +{ + return ((val) << A6XX_RB_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_RB_DEST_MSAA_CNTL_SAMPLES__MASK; +} +#define A6XX_RB_DEST_MSAA_CNTL_MSAA_DISABLE 0x00000004 + +#define REG_A6XX_RB_UNKNOWN_8804 0x00008804 + +#define REG_A6XX_RB_UNKNOWN_8805 0x00008805 + +#define REG_A6XX_RB_UNKNOWN_8806 0x00008806 + +#define REG_A6XX_RB_RENDER_CONTROL0 0x00008809 +#define A6XX_RB_RENDER_CONTROL0_VARYING 0x00000001 +#define A6XX_RB_RENDER_CONTROL0_XCOORD 0x00000040 +#define A6XX_RB_RENDER_CONTROL0_YCOORD 0x00000080 +#define A6XX_RB_RENDER_CONTROL0_ZCOORD 0x00000100 +#define A6XX_RB_RENDER_CONTROL0_WCOORD 0x00000200 +#define A6XX_RB_RENDER_CONTROL0_UNK10 0x00000400 + +#define REG_A6XX_RB_RENDER_CONTROL1 0x0000880a +#define A6XX_RB_RENDER_CONTROL1_SAMPLEMASK 0x00000001 +#define A6XX_RB_RENDER_CONTROL1_FACENESS 0x00000002 +#define A6XX_RB_RENDER_CONTROL1_SAMPLEID 0x00000008 + +#define REG_A6XX_RB_FS_OUTPUT_CNTL0 0x0000880b +#define A6XX_RB_FS_OUTPUT_CNTL0_FRAG_WRITES_Z 0x00000002 + +#define REG_A6XX_RB_FS_OUTPUT_CNTL1 0x0000880c +#define A6XX_RB_FS_OUTPUT_CNTL1_MRT__MASK 0x0000000f +#define A6XX_RB_FS_OUTPUT_CNTL1_MRT__SHIFT 0 +static inline uint32_t A6XX_RB_FS_OUTPUT_CNTL1_MRT(uint32_t val) +{ + return ((val) << A6XX_RB_FS_OUTPUT_CNTL1_MRT__SHIFT) & A6XX_RB_FS_OUTPUT_CNTL1_MRT__MASK; +} + +#define REG_A6XX_RB_RENDER_COMPONENTS 0x0000880d +#define A6XX_RB_RENDER_COMPONENTS_RT0__MASK 0x0000000f +#define A6XX_RB_RENDER_COMPONENTS_RT0__SHIFT 0 +static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT0(uint32_t val) +{ + return ((val) << A6XX_RB_RENDER_COMPONENTS_RT0__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT0__MASK; +} +#define A6XX_RB_RENDER_COMPONENTS_RT1__MASK 0x000000f0 +#define A6XX_RB_RENDER_COMPONENTS_RT1__SHIFT 4 +static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT1(uint32_t val) +{ + return ((val) << A6XX_RB_RENDER_COMPONENTS_RT1__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT1__MASK; +} +#define A6XX_RB_RENDER_COMPONENTS_RT2__MASK 0x00000f00 +#define A6XX_RB_RENDER_COMPONENTS_RT2__SHIFT 8 +static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT2(uint32_t val) +{ + return ((val) << A6XX_RB_RENDER_COMPONENTS_RT2__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT2__MASK; +} +#define A6XX_RB_RENDER_COMPONENTS_RT3__MASK 0x0000f000 +#define A6XX_RB_RENDER_COMPONENTS_RT3__SHIFT 12 +static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT3(uint32_t val) +{ + return ((val) << A6XX_RB_RENDER_COMPONENTS_RT3__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT3__MASK; +} +#define A6XX_RB_RENDER_COMPONENTS_RT4__MASK 0x000f0000 +#define A6XX_RB_RENDER_COMPONENTS_RT4__SHIFT 16 +static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT4(uint32_t val) +{ + return ((val) << A6XX_RB_RENDER_COMPONENTS_RT4__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT4__MASK; +} +#define A6XX_RB_RENDER_COMPONENTS_RT5__MASK 0x00f00000 +#define A6XX_RB_RENDER_COMPONENTS_RT5__SHIFT 20 +static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT5(uint32_t val) +{ + return ((val) << A6XX_RB_RENDER_COMPONENTS_RT5__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT5__MASK; +} +#define A6XX_RB_RENDER_COMPONENTS_RT6__MASK 0x0f000000 +#define A6XX_RB_RENDER_COMPONENTS_RT6__SHIFT 24 +static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT6(uint32_t val) +{ + return ((val) << A6XX_RB_RENDER_COMPONENTS_RT6__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT6__MASK; +} +#define A6XX_RB_RENDER_COMPONENTS_RT7__MASK 0xf0000000 +#define A6XX_RB_RENDER_COMPONENTS_RT7__SHIFT 28 +static inline uint32_t A6XX_RB_RENDER_COMPONENTS_RT7(uint32_t val) +{ + return ((val) << A6XX_RB_RENDER_COMPONENTS_RT7__SHIFT) & A6XX_RB_RENDER_COMPONENTS_RT7__MASK; +} + +#define REG_A6XX_RB_DITHER_CNTL 0x0000880e +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__MASK 0x00000003 +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__SHIFT 0 +static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0(enum adreno_rb_dither_mode val) +{ + return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT0__MASK; +} +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__MASK 0x0000000c +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__SHIFT 2 +static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1(enum adreno_rb_dither_mode val) +{ + return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT1__MASK; +} +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__MASK 0x00000030 +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__SHIFT 4 +static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2(enum adreno_rb_dither_mode val) +{ + return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT2__MASK; +} +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__MASK 0x000000c0 +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__SHIFT 6 +static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3(enum adreno_rb_dither_mode val) +{ + return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT3__MASK; +} +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__MASK 0x00000300 +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__SHIFT 8 +static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4(enum adreno_rb_dither_mode val) +{ + return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT4__MASK; +} +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__MASK 0x00000c00 +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__SHIFT 10 +static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5(enum adreno_rb_dither_mode val) +{ + return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT5__MASK; +} +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__MASK 0x00001000 +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__SHIFT 12 +static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6(enum adreno_rb_dither_mode val) +{ + return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT6__MASK; +} +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__MASK 0x0000c000 +#define A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__SHIFT 14 +static inline uint32_t A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7(enum adreno_rb_dither_mode val) +{ + return ((val) << A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__SHIFT) & A6XX_RB_DITHER_CNTL_DITHER_MODE_MRT7__MASK; +} + +#define REG_A6XX_RB_SRGB_CNTL 0x0000880f +#define A6XX_RB_SRGB_CNTL_SRGB_MRT0 0x00000001 +#define A6XX_RB_SRGB_CNTL_SRGB_MRT1 0x00000002 +#define A6XX_RB_SRGB_CNTL_SRGB_MRT2 0x00000004 +#define A6XX_RB_SRGB_CNTL_SRGB_MRT3 0x00000008 +#define A6XX_RB_SRGB_CNTL_SRGB_MRT4 0x00000010 +#define A6XX_RB_SRGB_CNTL_SRGB_MRT5 0x00000020 +#define A6XX_RB_SRGB_CNTL_SRGB_MRT6 0x00000040 +#define A6XX_RB_SRGB_CNTL_SRGB_MRT7 0x00000080 + +#define REG_A6XX_RB_UNKNOWN_8818 0x00008818 + +#define REG_A6XX_RB_UNKNOWN_8819 0x00008819 + +#define REG_A6XX_RB_UNKNOWN_881A 0x0000881a + +#define REG_A6XX_RB_UNKNOWN_881B 0x0000881b + +#define REG_A6XX_RB_UNKNOWN_881C 0x0000881c + +#define REG_A6XX_RB_UNKNOWN_881D 0x0000881d + +#define REG_A6XX_RB_UNKNOWN_881E 0x0000881e + +static inline uint32_t REG_A6XX_RB_MRT(uint32_t i0) { return 0x00008820 + 0x8*i0; } + +static inline uint32_t REG_A6XX_RB_MRT_CONTROL(uint32_t i0) { return 0x00008820 + 0x8*i0; } +#define A6XX_RB_MRT_CONTROL_BLEND 0x00000001 +#define A6XX_RB_MRT_CONTROL_BLEND2 0x00000002 +#define A6XX_RB_MRT_CONTROL_ROP_ENABLE 0x00000004 +#define A6XX_RB_MRT_CONTROL_ROP_CODE__MASK 0x00000078 +#define A6XX_RB_MRT_CONTROL_ROP_CODE__SHIFT 3 +static inline uint32_t A6XX_RB_MRT_CONTROL_ROP_CODE(enum a3xx_rop_code val) +{ + return ((val) << A6XX_RB_MRT_CONTROL_ROP_CODE__SHIFT) & A6XX_RB_MRT_CONTROL_ROP_CODE__MASK; +} +#define A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK 0x00000780 +#define A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT 7 +static inline uint32_t A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE(uint32_t val) +{ + return ((val) << A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__SHIFT) & A6XX_RB_MRT_CONTROL_COMPONENT_ENABLE__MASK; +} + +static inline uint32_t REG_A6XX_RB_MRT_BLEND_CONTROL(uint32_t i0) { return 0x00008821 + 0x8*i0; } +#define A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK 0x0000001f +#define A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT 0 +static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR(enum adreno_rb_blend_factor val) +{ + return ((val) << A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_RGB_SRC_FACTOR__MASK; +} +#define A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK 0x000000e0 +#define A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT 5 +static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE(enum a3xx_rb_blend_opcode val) +{ + return ((val) << A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_RGB_BLEND_OPCODE__MASK; +} +#define A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK 0x00001f00 +#define A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT 8 +static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR(enum adreno_rb_blend_factor val) +{ + return ((val) << A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_RGB_DEST_FACTOR__MASK; +} +#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK 0x001f0000 +#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT 16 +static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR(enum adreno_rb_blend_factor val) +{ + return ((val) << A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_ALPHA_SRC_FACTOR__MASK; +} +#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK 0x00e00000 +#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT 21 +static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE(enum a3xx_rb_blend_opcode val) +{ + return ((val) << A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_ALPHA_BLEND_OPCODE__MASK; +} +#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK 0x1f000000 +#define A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT 24 +static inline uint32_t A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR(enum adreno_rb_blend_factor val) +{ + return ((val) << A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__SHIFT) & A6XX_RB_MRT_BLEND_CONTROL_ALPHA_DEST_FACTOR__MASK; +} + +static inline uint32_t REG_A6XX_RB_MRT_BUF_INFO(uint32_t i0) { return 0x00008822 + 0x8*i0; } +#define A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK 0x000000ff +#define A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT 0 +static inline uint32_t A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT(enum a6xx_color_fmt val) +{ + return ((val) << A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__SHIFT) & A6XX_RB_MRT_BUF_INFO_COLOR_FORMAT__MASK; +} +#define A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK 0x00000300 +#define A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT 8 +static inline uint32_t A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE(enum a6xx_tile_mode val) +{ + return ((val) << A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__SHIFT) & A6XX_RB_MRT_BUF_INFO_COLOR_TILE_MODE__MASK; +} +#define A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK 0x00006000 +#define A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT 13 +static inline uint32_t A6XX_RB_MRT_BUF_INFO_COLOR_SWAP(enum a3xx_color_swap val) +{ + return ((val) << A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__SHIFT) & A6XX_RB_MRT_BUF_INFO_COLOR_SWAP__MASK; +} +#define A6XX_RB_MRT_BUF_INFO_COLOR_SRGB 0x00008000 + +static inline uint32_t REG_A6XX_RB_MRT_PITCH(uint32_t i0) { return 0x00008823 + 0x8*i0; } +#define A6XX_RB_MRT_PITCH__MASK 0xffffffff +#define A6XX_RB_MRT_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_MRT_PITCH(uint32_t val) +{ + return ((val >> 6) << A6XX_RB_MRT_PITCH__SHIFT) & A6XX_RB_MRT_PITCH__MASK; +} + +static inline uint32_t REG_A6XX_RB_MRT_ARRAY_PITCH(uint32_t i0) { return 0x00008824 + 0x8*i0; } +#define A6XX_RB_MRT_ARRAY_PITCH__MASK 0xffffffff +#define A6XX_RB_MRT_ARRAY_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_MRT_ARRAY_PITCH(uint32_t val) +{ + return ((val >> 6) << A6XX_RB_MRT_ARRAY_PITCH__SHIFT) & A6XX_RB_MRT_ARRAY_PITCH__MASK; +} + +static inline uint32_t REG_A6XX_RB_MRT_BASE_LO(uint32_t i0) { return 0x00008825 + 0x8*i0; } + +static inline uint32_t REG_A6XX_RB_MRT_BASE_HI(uint32_t i0) { return 0x00008826 + 0x8*i0; } + +static inline uint32_t REG_A6XX_RB_MRT_BASE_GMEM(uint32_t i0) { return 0x00008827 + 0x8*i0; } + +#define REG_A6XX_RB_BLEND_RED_F32 0x00008860 +#define A6XX_RB_BLEND_RED_F32__MASK 0xffffffff +#define A6XX_RB_BLEND_RED_F32__SHIFT 0 +static inline uint32_t A6XX_RB_BLEND_RED_F32(float val) +{ + return ((fui(val)) << A6XX_RB_BLEND_RED_F32__SHIFT) & A6XX_RB_BLEND_RED_F32__MASK; +} + +#define REG_A6XX_RB_BLEND_GREEN_F32 0x00008861 +#define A6XX_RB_BLEND_GREEN_F32__MASK 0xffffffff +#define A6XX_RB_BLEND_GREEN_F32__SHIFT 0 +static inline uint32_t A6XX_RB_BLEND_GREEN_F32(float val) +{ + return ((fui(val)) << A6XX_RB_BLEND_GREEN_F32__SHIFT) & A6XX_RB_BLEND_GREEN_F32__MASK; +} + +#define REG_A6XX_RB_BLEND_BLUE_F32 0x00008862 +#define A6XX_RB_BLEND_BLUE_F32__MASK 0xffffffff +#define A6XX_RB_BLEND_BLUE_F32__SHIFT 0 +static inline uint32_t A6XX_RB_BLEND_BLUE_F32(float val) +{ + return ((fui(val)) << A6XX_RB_BLEND_BLUE_F32__SHIFT) & A6XX_RB_BLEND_BLUE_F32__MASK; +} + +#define REG_A6XX_RB_BLEND_ALPHA_F32 0x00008863 +#define A6XX_RB_BLEND_ALPHA_F32__MASK 0xffffffff +#define A6XX_RB_BLEND_ALPHA_F32__SHIFT 0 +static inline uint32_t A6XX_RB_BLEND_ALPHA_F32(float val) +{ + return ((fui(val)) << A6XX_RB_BLEND_ALPHA_F32__SHIFT) & A6XX_RB_BLEND_ALPHA_F32__MASK; +} + +#define REG_A6XX_RB_ALPHA_CONTROL 0x00008864 +#define A6XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK 0x000000ff +#define A6XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT 0 +static inline uint32_t A6XX_RB_ALPHA_CONTROL_ALPHA_REF(uint32_t val) +{ + return ((val) << A6XX_RB_ALPHA_CONTROL_ALPHA_REF__SHIFT) & A6XX_RB_ALPHA_CONTROL_ALPHA_REF__MASK; +} +#define A6XX_RB_ALPHA_CONTROL_ALPHA_TEST 0x00000100 +#define A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK 0x00000e00 +#define A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT 9 +static inline uint32_t A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC(enum adreno_compare_func val) +{ + return ((val) << A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__SHIFT) & A6XX_RB_ALPHA_CONTROL_ALPHA_TEST_FUNC__MASK; +} + +#define REG_A6XX_RB_BLEND_CNTL 0x00008865 +#define A6XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK 0x000000ff +#define A6XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT 0 +static inline uint32_t A6XX_RB_BLEND_CNTL_ENABLE_BLEND(uint32_t val) +{ + return ((val) << A6XX_RB_BLEND_CNTL_ENABLE_BLEND__SHIFT) & A6XX_RB_BLEND_CNTL_ENABLE_BLEND__MASK; +} +#define A6XX_RB_BLEND_CNTL_INDEPENDENT_BLEND 0x00000100 +#define A6XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK 0xffff0000 +#define A6XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT 16 +static inline uint32_t A6XX_RB_BLEND_CNTL_SAMPLE_MASK(uint32_t val) +{ + return ((val) << A6XX_RB_BLEND_CNTL_SAMPLE_MASK__SHIFT) & A6XX_RB_BLEND_CNTL_SAMPLE_MASK__MASK; +} + +#define REG_A6XX_RB_DEPTH_CNTL 0x00008871 +#define A6XX_RB_DEPTH_CNTL_Z_ENABLE 0x00000001 +#define A6XX_RB_DEPTH_CNTL_Z_WRITE_ENABLE 0x00000002 +#define A6XX_RB_DEPTH_CNTL_ZFUNC__MASK 0x0000001c +#define A6XX_RB_DEPTH_CNTL_ZFUNC__SHIFT 2 +static inline uint32_t A6XX_RB_DEPTH_CNTL_ZFUNC(enum adreno_compare_func val) +{ + return ((val) << A6XX_RB_DEPTH_CNTL_ZFUNC__SHIFT) & A6XX_RB_DEPTH_CNTL_ZFUNC__MASK; +} +#define A6XX_RB_DEPTH_CNTL_Z_TEST_ENABLE 0x00000040 + +#define REG_A6XX_RB_DEPTH_BUFFER_INFO 0x00008872 +#define A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK 0x00000007 +#define A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT 0 +static inline uint32_t A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT(enum a6xx_depth_format val) +{ + return ((val) << A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__SHIFT) & A6XX_RB_DEPTH_BUFFER_INFO_DEPTH_FORMAT__MASK; +} + +#define REG_A6XX_RB_DEPTH_BUFFER_PITCH 0x00008873 +#define A6XX_RB_DEPTH_BUFFER_PITCH__MASK 0xffffffff +#define A6XX_RB_DEPTH_BUFFER_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_DEPTH_BUFFER_PITCH(uint32_t val) +{ + return ((val >> 6) << A6XX_RB_DEPTH_BUFFER_PITCH__SHIFT) & A6XX_RB_DEPTH_BUFFER_PITCH__MASK; +} + +#define REG_A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH 0x00008874 +#define A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK 0xffffffff +#define A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH(uint32_t val) +{ + return ((val >> 6) << A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__SHIFT) & A6XX_RB_DEPTH_BUFFER_ARRAY_PITCH__MASK; +} + +#define REG_A6XX_RB_DEPTH_BUFFER_BASE_LO 0x00008875 + +#define REG_A6XX_RB_DEPTH_BUFFER_BASE_HI 0x00008876 + +#define REG_A6XX_RB_DEPTH_BUFFER_BASE_GMEM 0x00008877 + +#define REG_A6XX_RB_UNKNOWN_8878 0x00008878 + +#define REG_A6XX_RB_UNKNOWN_8879 0x00008879 + +#define REG_A6XX_RB_STENCIL_CONTROL 0x00008880 +#define A6XX_RB_STENCIL_CONTROL_STENCIL_ENABLE 0x00000001 +#define A6XX_RB_STENCIL_CONTROL_STENCIL_ENABLE_BF 0x00000002 +#define A6XX_RB_STENCIL_CONTROL_STENCIL_READ 0x00000004 +#define A6XX_RB_STENCIL_CONTROL_FUNC__MASK 0x00000700 +#define A6XX_RB_STENCIL_CONTROL_FUNC__SHIFT 8 +static inline uint32_t A6XX_RB_STENCIL_CONTROL_FUNC(enum adreno_compare_func val) +{ + return ((val) << A6XX_RB_STENCIL_CONTROL_FUNC__SHIFT) & A6XX_RB_STENCIL_CONTROL_FUNC__MASK; +} +#define A6XX_RB_STENCIL_CONTROL_FAIL__MASK 0x00003800 +#define A6XX_RB_STENCIL_CONTROL_FAIL__SHIFT 11 +static inline uint32_t A6XX_RB_STENCIL_CONTROL_FAIL(enum adreno_stencil_op val) +{ + return ((val) << A6XX_RB_STENCIL_CONTROL_FAIL__SHIFT) & A6XX_RB_STENCIL_CONTROL_FAIL__MASK; +} +#define A6XX_RB_STENCIL_CONTROL_ZPASS__MASK 0x0001c000 +#define A6XX_RB_STENCIL_CONTROL_ZPASS__SHIFT 14 +static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZPASS(enum adreno_stencil_op val) +{ + return ((val) << A6XX_RB_STENCIL_CONTROL_ZPASS__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZPASS__MASK; +} +#define A6XX_RB_STENCIL_CONTROL_ZFAIL__MASK 0x000e0000 +#define A6XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT 17 +static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZFAIL(enum adreno_stencil_op val) +{ + return ((val) << A6XX_RB_STENCIL_CONTROL_ZFAIL__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZFAIL__MASK; +} +#define A6XX_RB_STENCIL_CONTROL_FUNC_BF__MASK 0x00700000 +#define A6XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT 20 +static inline uint32_t A6XX_RB_STENCIL_CONTROL_FUNC_BF(enum adreno_compare_func val) +{ + return ((val) << A6XX_RB_STENCIL_CONTROL_FUNC_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_FUNC_BF__MASK; +} +#define A6XX_RB_STENCIL_CONTROL_FAIL_BF__MASK 0x03800000 +#define A6XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT 23 +static inline uint32_t A6XX_RB_STENCIL_CONTROL_FAIL_BF(enum adreno_stencil_op val) +{ + return ((val) << A6XX_RB_STENCIL_CONTROL_FAIL_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_FAIL_BF__MASK; +} +#define A6XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK 0x1c000000 +#define A6XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT 26 +static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZPASS_BF(enum adreno_stencil_op val) +{ + return ((val) << A6XX_RB_STENCIL_CONTROL_ZPASS_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZPASS_BF__MASK; +} +#define A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK 0xe0000000 +#define A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT 29 +static inline uint32_t A6XX_RB_STENCIL_CONTROL_ZFAIL_BF(enum adreno_stencil_op val) +{ + return ((val) << A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__SHIFT) & A6XX_RB_STENCIL_CONTROL_ZFAIL_BF__MASK; +} + +#define REG_A6XX_RB_STENCIL_INFO 0x00008881 +#define A6XX_RB_STENCIL_INFO_SEPARATE_STENCIL 0x00000001 + +#define REG_A6XX_RB_STENCIL_BUFFER_PITCH 0x00008882 +#define A6XX_RB_STENCIL_BUFFER_PITCH__MASK 0xffffffff +#define A6XX_RB_STENCIL_BUFFER_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_STENCIL_BUFFER_PITCH(uint32_t val) +{ + return ((val >> 6) << A6XX_RB_STENCIL_BUFFER_PITCH__SHIFT) & A6XX_RB_STENCIL_BUFFER_PITCH__MASK; +} + +#define REG_A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH 0x00008883 +#define A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__MASK 0xffffffff +#define A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH(uint32_t val) +{ + return ((val >> 6) << A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__SHIFT) & A6XX_RB_STENCIL_BUFFER_ARRAY_PITCH__MASK; +} + +#define REG_A6XX_RB_STENCIL_BUFFER_BASE_LO 0x00008884 + +#define REG_A6XX_RB_STENCIL_BUFFER_BASE_HI 0x00008885 + +#define REG_A6XX_RB_STENCIL_BUFFER_BASE_GMEM 0x00008886 + +#define REG_A6XX_RB_STENCILREF 0x00008887 +#define A6XX_RB_STENCILREF_REF__MASK 0x000000ff +#define A6XX_RB_STENCILREF_REF__SHIFT 0 +static inline uint32_t A6XX_RB_STENCILREF_REF(uint32_t val) +{ + return ((val) << A6XX_RB_STENCILREF_REF__SHIFT) & A6XX_RB_STENCILREF_REF__MASK; +} + +#define REG_A6XX_RB_STENCILMASK 0x00008888 +#define A6XX_RB_STENCILMASK_MASK__MASK 0x000000ff +#define A6XX_RB_STENCILMASK_MASK__SHIFT 0 +static inline uint32_t A6XX_RB_STENCILMASK_MASK(uint32_t val) +{ + return ((val) << A6XX_RB_STENCILMASK_MASK__SHIFT) & A6XX_RB_STENCILMASK_MASK__MASK; +} + +#define REG_A6XX_RB_STENCILWRMASK 0x00008889 +#define A6XX_RB_STENCILWRMASK_WRMASK__MASK 0x000000ff +#define A6XX_RB_STENCILWRMASK_WRMASK__SHIFT 0 +static inline uint32_t A6XX_RB_STENCILWRMASK_WRMASK(uint32_t val) +{ + return ((val) << A6XX_RB_STENCILWRMASK_WRMASK__SHIFT) & A6XX_RB_STENCILWRMASK_WRMASK__MASK; +} + +#define REG_A6XX_RB_WINDOW_OFFSET 0x00008890 +#define A6XX_RB_WINDOW_OFFSET_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_RB_WINDOW_OFFSET_X__MASK 0x00007fff +#define A6XX_RB_WINDOW_OFFSET_X__SHIFT 0 +static inline uint32_t A6XX_RB_WINDOW_OFFSET_X(uint32_t val) +{ + return ((val) << A6XX_RB_WINDOW_OFFSET_X__SHIFT) & A6XX_RB_WINDOW_OFFSET_X__MASK; +} +#define A6XX_RB_WINDOW_OFFSET_Y__MASK 0x7fff0000 +#define A6XX_RB_WINDOW_OFFSET_Y__SHIFT 16 +static inline uint32_t A6XX_RB_WINDOW_OFFSET_Y(uint32_t val) +{ + return ((val) << A6XX_RB_WINDOW_OFFSET_Y__SHIFT) & A6XX_RB_WINDOW_OFFSET_Y__MASK; +} + +#define REG_A6XX_RB_SAMPLE_COUNT_CONTROL 0x00008891 +#define A6XX_RB_SAMPLE_COUNT_CONTROL_COPY 0x00000002 + +#define REG_A6XX_RB_UNKNOWN_88D0 0x000088d0 + +#define REG_A6XX_RB_BLIT_SCISSOR_TL 0x000088d1 +#define A6XX_RB_BLIT_SCISSOR_TL_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_RB_BLIT_SCISSOR_TL_X__MASK 0x00007fff +#define A6XX_RB_BLIT_SCISSOR_TL_X__SHIFT 0 +static inline uint32_t A6XX_RB_BLIT_SCISSOR_TL_X(uint32_t val) +{ + return ((val) << A6XX_RB_BLIT_SCISSOR_TL_X__SHIFT) & A6XX_RB_BLIT_SCISSOR_TL_X__MASK; +} +#define A6XX_RB_BLIT_SCISSOR_TL_Y__MASK 0x7fff0000 +#define A6XX_RB_BLIT_SCISSOR_TL_Y__SHIFT 16 +static inline uint32_t A6XX_RB_BLIT_SCISSOR_TL_Y(uint32_t val) +{ + return ((val) << A6XX_RB_BLIT_SCISSOR_TL_Y__SHIFT) & A6XX_RB_BLIT_SCISSOR_TL_Y__MASK; +} + +#define REG_A6XX_RB_BLIT_SCISSOR_BR 0x000088d2 +#define A6XX_RB_BLIT_SCISSOR_BR_WINDOW_OFFSET_DISABLE 0x80000000 +#define A6XX_RB_BLIT_SCISSOR_BR_X__MASK 0x00007fff +#define A6XX_RB_BLIT_SCISSOR_BR_X__SHIFT 0 +static inline uint32_t A6XX_RB_BLIT_SCISSOR_BR_X(uint32_t val) +{ + return ((val) << A6XX_RB_BLIT_SCISSOR_BR_X__SHIFT) & A6XX_RB_BLIT_SCISSOR_BR_X__MASK; +} +#define A6XX_RB_BLIT_SCISSOR_BR_Y__MASK 0x7fff0000 +#define A6XX_RB_BLIT_SCISSOR_BR_Y__SHIFT 16 +static inline uint32_t A6XX_RB_BLIT_SCISSOR_BR_Y(uint32_t val) +{ + return ((val) << A6XX_RB_BLIT_SCISSOR_BR_Y__SHIFT) & A6XX_RB_BLIT_SCISSOR_BR_Y__MASK; +} + +#define REG_A6XX_RB_BLIT_BASE_GMEM 0x000088d6 + +#define REG_A6XX_RB_BLIT_DST_INFO 0x000088d7 +#define A6XX_RB_BLIT_DST_INFO_TILE_MODE__MASK 0x00000003 +#define A6XX_RB_BLIT_DST_INFO_TILE_MODE__SHIFT 0 +static inline uint32_t A6XX_RB_BLIT_DST_INFO_TILE_MODE(enum a6xx_tile_mode val) +{ + return ((val) << A6XX_RB_BLIT_DST_INFO_TILE_MODE__SHIFT) & A6XX_RB_BLIT_DST_INFO_TILE_MODE__MASK; +} +#define A6XX_RB_BLIT_DST_INFO_FLAGS 0x00000004 +#define A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__MASK 0x00007f80 +#define A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__SHIFT 7 +static inline uint32_t A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT(enum a6xx_color_fmt val) +{ + return ((val) << A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__SHIFT) & A6XX_RB_BLIT_DST_INFO_COLOR_FORMAT__MASK; +} +#define A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__MASK 0x00000060 +#define A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__SHIFT 5 +static inline uint32_t A6XX_RB_BLIT_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val) +{ + return ((val) << A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__SHIFT) & A6XX_RB_BLIT_DST_INFO_COLOR_SWAP__MASK; +} + +#define REG_A6XX_RB_BLIT_DST_LO 0x000088d8 + +#define REG_A6XX_RB_BLIT_DST_HI 0x000088d9 + +#define REG_A6XX_RB_BLIT_DST_PITCH 0x000088da +#define A6XX_RB_BLIT_DST_PITCH__MASK 0xffffffff +#define A6XX_RB_BLIT_DST_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_BLIT_DST_PITCH(uint32_t val) +{ + return ((val >> 6) << A6XX_RB_BLIT_DST_PITCH__SHIFT) & A6XX_RB_BLIT_DST_PITCH__MASK; +} + +#define REG_A6XX_RB_BLIT_DST_ARRAY_PITCH 0x000088db +#define A6XX_RB_BLIT_DST_ARRAY_PITCH__MASK 0xffffffff +#define A6XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_BLIT_DST_ARRAY_PITCH(uint32_t val) +{ + return ((val >> 6) << A6XX_RB_BLIT_DST_ARRAY_PITCH__SHIFT) & A6XX_RB_BLIT_DST_ARRAY_PITCH__MASK; +} + +#define REG_A6XX_RB_BLIT_FLAG_DST_LO 0x000088dc + +#define REG_A6XX_RB_BLIT_FLAG_DST_HI 0x000088dd + +#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW0 0x000088df + +#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW1 0x000088e0 + +#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW2 0x000088e1 + +#define REG_A6XX_RB_BLIT_CLEAR_COLOR_DW3 0x000088e2 + +#define REG_A6XX_RB_BLIT_INFO 0x000088e3 +#define A6XX_RB_BLIT_INFO_UNK0 0x00000001 +#define A6XX_RB_BLIT_INFO_FAST_CLEAR 0x00000002 +#define A6XX_RB_BLIT_INFO_INTEGER 0x00000004 +#define A6XX_RB_BLIT_INFO_UNK3 0x00000008 +#define A6XX_RB_BLIT_INFO_MASK__MASK 0x000000f0 +#define A6XX_RB_BLIT_INFO_MASK__SHIFT 4 +static inline uint32_t A6XX_RB_BLIT_INFO_MASK(uint32_t val) +{ + return ((val) << A6XX_RB_BLIT_INFO_MASK__SHIFT) & A6XX_RB_BLIT_INFO_MASK__MASK; +} + +#define REG_A6XX_RB_UNKNOWN_88F0 0x000088f0 + +#define REG_A6XX_RB_DEPTH_FLAG_BUFFER_BASE_LO 0x00008900 + +#define REG_A6XX_RB_DEPTH_FLAG_BUFFER_BASE_HI 0x00008901 + +#define REG_A6XX_RB_DEPTH_FLAG_BUFFER_PITCH 0x00008902 + +static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER(uint32_t i0) { return 0x00008903 + 0x3*i0; } + +static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER_ADDR_LO(uint32_t i0) { return 0x00008903 + 0x3*i0; } + +static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER_ADDR_HI(uint32_t i0) { return 0x00008904 + 0x3*i0; } + +static inline uint32_t REG_A6XX_RB_MRT_FLAG_BUFFER_PITCH(uint32_t i0) { return 0x00008905 + 0x3*i0; } +#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__MASK 0x000007ff +#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH(uint32_t val) +{ + return ((val >> 5) << A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__SHIFT) & A6XX_RB_MRT_FLAG_BUFFER_PITCH_PITCH__MASK; +} +#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK 0x003ff800 +#define A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT 11 +static inline uint32_t A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH(uint32_t val) +{ + return ((val >> 5) << A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__SHIFT) & A6XX_RB_MRT_FLAG_BUFFER_PITCH_ARRAY_PITCH__MASK; +} + +#define REG_A6XX_RB_SAMPLE_COUNT_ADDR_LO 0x00008927 + +#define REG_A6XX_RB_SAMPLE_COUNT_ADDR_HI 0x00008928 + +#define REG_A6XX_RB_2D_BLIT_CNTL 0x00008c00 +#define A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__MASK 0x0000ff00 +#define A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__SHIFT 8 +static inline uint32_t A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT(enum a6xx_color_fmt val) +{ + return ((val) << A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__SHIFT) & A6XX_RB_2D_BLIT_CNTL_COLOR_FORMAT__MASK; +} + +#define REG_A6XX_RB_2D_DST_INFO 0x00008c17 +#define A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK 0x000000ff +#define A6XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT 0 +static inline uint32_t A6XX_RB_2D_DST_INFO_COLOR_FORMAT(enum a6xx_color_fmt val) +{ + return ((val) << A6XX_RB_2D_DST_INFO_COLOR_FORMAT__SHIFT) & A6XX_RB_2D_DST_INFO_COLOR_FORMAT__MASK; +} +#define A6XX_RB_2D_DST_INFO_TILE_MODE__MASK 0x00000300 +#define A6XX_RB_2D_DST_INFO_TILE_MODE__SHIFT 8 +static inline uint32_t A6XX_RB_2D_DST_INFO_TILE_MODE(enum a6xx_tile_mode val) +{ + return ((val) << A6XX_RB_2D_DST_INFO_TILE_MODE__SHIFT) & A6XX_RB_2D_DST_INFO_TILE_MODE__MASK; +} +#define A6XX_RB_2D_DST_INFO_COLOR_SWAP__MASK 0x00000c00 +#define A6XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT 10 +static inline uint32_t A6XX_RB_2D_DST_INFO_COLOR_SWAP(enum a3xx_color_swap val) +{ + return ((val) << A6XX_RB_2D_DST_INFO_COLOR_SWAP__SHIFT) & A6XX_RB_2D_DST_INFO_COLOR_SWAP__MASK; +} +#define A6XX_RB_2D_DST_INFO_FLAGS 0x00001000 + +#define REG_A6XX_RB_2D_DST_LO 0x00008c18 + +#define REG_A6XX_RB_2D_DST_HI 0x00008c19 + +#define REG_A6XX_RB_2D_DST_SIZE 0x00008c1a +#define A6XX_RB_2D_DST_SIZE_PITCH__MASK 0x0000ffff +#define A6XX_RB_2D_DST_SIZE_PITCH__SHIFT 0 +static inline uint32_t A6XX_RB_2D_DST_SIZE_PITCH(uint32_t val) +{ + return ((val >> 6) << A6XX_RB_2D_DST_SIZE_PITCH__SHIFT) & A6XX_RB_2D_DST_SIZE_PITCH__MASK; +} + +#define REG_A6XX_RB_2D_DST_FLAGS_LO 0x00008c20 + +#define REG_A6XX_RB_2D_DST_FLAGS_HI 0x00008c21 + +#define REG_A6XX_RB_2D_SRC_SOLID_C0 0x00008c2c + +#define REG_A6XX_RB_2D_SRC_SOLID_C1 0x00008c2d + +#define REG_A6XX_RB_2D_SRC_SOLID_C2 0x00008c2e + +#define REG_A6XX_RB_2D_SRC_SOLID_C3 0x00008c2f + +#define REG_A6XX_RB_UNKNOWN_8E01 0x00008e01 + +#define REG_A6XX_RB_CCU_CNTL 0x00008e07 + +#define REG_A6XX_VPC_UNKNOWN_9101 0x00009101 + +#define REG_A6XX_VPC_GS_SIV_CNTL 0x00009104 + +#define REG_A6XX_VPC_UNKNOWN_9108 0x00009108 + +static inline uint32_t REG_A6XX_VPC_VARYING_INTERP(uint32_t i0) { return 0x00009200 + 0x1*i0; } + +static inline uint32_t REG_A6XX_VPC_VARYING_INTERP_MODE(uint32_t i0) { return 0x00009200 + 0x1*i0; } + +static inline uint32_t REG_A6XX_VPC_VARYING_PS_REPL(uint32_t i0) { return 0x00009208 + 0x1*i0; } + +static inline uint32_t REG_A6XX_VPC_VARYING_PS_REPL_MODE(uint32_t i0) { return 0x00009208 + 0x1*i0; } + +#define REG_A6XX_VPC_UNKNOWN_9210 0x00009210 + +#define REG_A6XX_VPC_UNKNOWN_9211 0x00009211 + +static inline uint32_t REG_A6XX_VPC_VAR(uint32_t i0) { return 0x00009212 + 0x1*i0; } + +static inline uint32_t REG_A6XX_VPC_VAR_DISABLE(uint32_t i0) { return 0x00009212 + 0x1*i0; } + +#define REG_A6XX_VPC_SO_CNTL 0x00009216 +#define A6XX_VPC_SO_CNTL_ENABLE 0x00010000 + +#define REG_A6XX_VPC_SO_PROG 0x00009217 +#define A6XX_VPC_SO_PROG_A_BUF__MASK 0x00000003 +#define A6XX_VPC_SO_PROG_A_BUF__SHIFT 0 +static inline uint32_t A6XX_VPC_SO_PROG_A_BUF(uint32_t val) +{ + return ((val) << A6XX_VPC_SO_PROG_A_BUF__SHIFT) & A6XX_VPC_SO_PROG_A_BUF__MASK; +} +#define A6XX_VPC_SO_PROG_A_OFF__MASK 0x000007fc +#define A6XX_VPC_SO_PROG_A_OFF__SHIFT 2 +static inline uint32_t A6XX_VPC_SO_PROG_A_OFF(uint32_t val) +{ + return ((val >> 2) << A6XX_VPC_SO_PROG_A_OFF__SHIFT) & A6XX_VPC_SO_PROG_A_OFF__MASK; +} +#define A6XX_VPC_SO_PROG_A_EN 0x00000800 +#define A6XX_VPC_SO_PROG_B_BUF__MASK 0x00003000 +#define A6XX_VPC_SO_PROG_B_BUF__SHIFT 12 +static inline uint32_t A6XX_VPC_SO_PROG_B_BUF(uint32_t val) +{ + return ((val) << A6XX_VPC_SO_PROG_B_BUF__SHIFT) & A6XX_VPC_SO_PROG_B_BUF__MASK; +} +#define A6XX_VPC_SO_PROG_B_OFF__MASK 0x007fc000 +#define A6XX_VPC_SO_PROG_B_OFF__SHIFT 14 +static inline uint32_t A6XX_VPC_SO_PROG_B_OFF(uint32_t val) +{ + return ((val >> 2) << A6XX_VPC_SO_PROG_B_OFF__SHIFT) & A6XX_VPC_SO_PROG_B_OFF__MASK; +} +#define A6XX_VPC_SO_PROG_B_EN 0x00800000 + +static inline uint32_t REG_A6XX_VPC_SO(uint32_t i0) { return 0x0000921a + 0x7*i0; } + +static inline uint32_t REG_A6XX_VPC_SO_BUFFER_BASE_LO(uint32_t i0) { return 0x0000921a + 0x7*i0; } + +static inline uint32_t REG_A6XX_VPC_SO_BUFFER_BASE_HI(uint32_t i0) { return 0x0000921b + 0x7*i0; } + +static inline uint32_t REG_A6XX_VPC_SO_BUFFER_SIZE(uint32_t i0) { return 0x0000921c + 0x7*i0; } + +static inline uint32_t REG_A6XX_VPC_SO_NCOMP(uint32_t i0) { return 0x0000921d + 0x7*i0; } + +static inline uint32_t REG_A6XX_VPC_SO_BUFFER_OFFSET(uint32_t i0) { return 0x0000921e + 0x7*i0; } + +static inline uint32_t REG_A6XX_VPC_SO_FLUSH_BASE_LO(uint32_t i0) { return 0x0000921f + 0x7*i0; } + +static inline uint32_t REG_A6XX_VPC_SO_FLUSH_BASE_HI(uint32_t i0) { return 0x00009220 + 0x7*i0; } + +#define REG_A6XX_VPC_UNKNOWN_9236 0x00009236 + +#define REG_A6XX_VPC_UNKNOWN_9300 0x00009300 + +#define REG_A6XX_VPC_PACK 0x00009301 +#define A6XX_VPC_PACK_STRIDE_IN_VPC__MASK 0x000000ff +#define A6XX_VPC_PACK_STRIDE_IN_VPC__SHIFT 0 +static inline uint32_t A6XX_VPC_PACK_STRIDE_IN_VPC(uint32_t val) +{ + return ((val) << A6XX_VPC_PACK_STRIDE_IN_VPC__SHIFT) & A6XX_VPC_PACK_STRIDE_IN_VPC__MASK; +} +#define A6XX_VPC_PACK_NUMNONPOSVAR__MASK 0x0000ff00 +#define A6XX_VPC_PACK_NUMNONPOSVAR__SHIFT 8 +static inline uint32_t A6XX_VPC_PACK_NUMNONPOSVAR(uint32_t val) +{ + return ((val) << A6XX_VPC_PACK_NUMNONPOSVAR__SHIFT) & A6XX_VPC_PACK_NUMNONPOSVAR__MASK; +} +#define A6XX_VPC_PACK_PSIZELOC__MASK 0x00ff0000 +#define A6XX_VPC_PACK_PSIZELOC__SHIFT 16 +static inline uint32_t A6XX_VPC_PACK_PSIZELOC(uint32_t val) +{ + return ((val) << A6XX_VPC_PACK_PSIZELOC__SHIFT) & A6XX_VPC_PACK_PSIZELOC__MASK; +} + +#define REG_A6XX_VPC_CNTL_0 0x00009304 +#define A6XX_VPC_CNTL_0_NUMNONPOSVAR__MASK 0x000000ff +#define A6XX_VPC_CNTL_0_NUMNONPOSVAR__SHIFT 0 +static inline uint32_t A6XX_VPC_CNTL_0_NUMNONPOSVAR(uint32_t val) +{ + return ((val) << A6XX_VPC_CNTL_0_NUMNONPOSVAR__SHIFT) & A6XX_VPC_CNTL_0_NUMNONPOSVAR__MASK; +} +#define A6XX_VPC_CNTL_0_VARYING 0x00010000 + +#define REG_A6XX_VPC_SO_BUF_CNTL 0x00009305 +#define A6XX_VPC_SO_BUF_CNTL_BUF0 0x00000001 +#define A6XX_VPC_SO_BUF_CNTL_BUF1 0x00000008 +#define A6XX_VPC_SO_BUF_CNTL_BUF2 0x00000040 +#define A6XX_VPC_SO_BUF_CNTL_BUF3 0x00000200 +#define A6XX_VPC_SO_BUF_CNTL_ENABLE 0x00008000 + +#define REG_A6XX_VPC_UNKNOWN_9600 0x00009600 + +#define REG_A6XX_VPC_UNKNOWN_9602 0x00009602 + +#define REG_A6XX_PC_UNKNOWN_9801 0x00009801 + +#define REG_A6XX_PC_RESTART_INDEX 0x00009803 + +#define REG_A6XX_PC_MODE_CNTL 0x00009804 + +#define REG_A6XX_PC_UNKNOWN_9805 0x00009805 + +#define REG_A6XX_PC_UNKNOWN_9981 0x00009981 + +#define REG_A6XX_PC_PRIMITIVE_CNTL_0 0x00009b00 +#define A6XX_PC_PRIMITIVE_CNTL_0_PRIMITIVE_RESTART 0x00000001 +#define A6XX_PC_PRIMITIVE_CNTL_0_PROVOKING_VTX_LAST 0x00000002 + +#define REG_A6XX_PC_PRIMITIVE_CNTL_1 0x00009b01 +#define A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC__MASK 0x0000007f +#define A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC__SHIFT 0 +static inline uint32_t A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC(uint32_t val) +{ + return ((val) << A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC__SHIFT) & A6XX_PC_PRIMITIVE_CNTL_1_STRIDE_IN_VPC__MASK; +} + +#define REG_A6XX_PC_UNKNOWN_9B06 0x00009b06 + +#define REG_A6XX_PC_UNKNOWN_9B07 0x00009b07 + +#define REG_A6XX_PC_TESSFACTOR_ADDR_LO 0x00009e08 + +#define REG_A6XX_PC_TESSFACTOR_ADDR_HI 0x00009e09 + +#define REG_A6XX_PC_UNKNOWN_9E72 0x00009e72 + +#define REG_A6XX_VFD_CONTROL_0 0x0000a000 +#define A6XX_VFD_CONTROL_0_VTXCNT__MASK 0x0000003f +#define A6XX_VFD_CONTROL_0_VTXCNT__SHIFT 0 +static inline uint32_t A6XX_VFD_CONTROL_0_VTXCNT(uint32_t val) +{ + return ((val) << A6XX_VFD_CONTROL_0_VTXCNT__SHIFT) & A6XX_VFD_CONTROL_0_VTXCNT__MASK; +} + +#define REG_A6XX_VFD_CONTROL_1 0x0000a001 +#define A6XX_VFD_CONTROL_1_REGID4VTX__MASK 0x000000ff +#define A6XX_VFD_CONTROL_1_REGID4VTX__SHIFT 0 +static inline uint32_t A6XX_VFD_CONTROL_1_REGID4VTX(uint32_t val) +{ + return ((val) << A6XX_VFD_CONTROL_1_REGID4VTX__SHIFT) & A6XX_VFD_CONTROL_1_REGID4VTX__MASK; +} +#define A6XX_VFD_CONTROL_1_REGID4INST__MASK 0x0000ff00 +#define A6XX_VFD_CONTROL_1_REGID4INST__SHIFT 8 +static inline uint32_t A6XX_VFD_CONTROL_1_REGID4INST(uint32_t val) +{ + return ((val) << A6XX_VFD_CONTROL_1_REGID4INST__SHIFT) & A6XX_VFD_CONTROL_1_REGID4INST__MASK; +} +#define A6XX_VFD_CONTROL_1_REGID4PRIMID__MASK 0x00ff0000 +#define A6XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT 16 +static inline uint32_t A6XX_VFD_CONTROL_1_REGID4PRIMID(uint32_t val) +{ + return ((val) << A6XX_VFD_CONTROL_1_REGID4PRIMID__SHIFT) & A6XX_VFD_CONTROL_1_REGID4PRIMID__MASK; +} + +#define REG_A6XX_VFD_CONTROL_2 0x0000a002 +#define A6XX_VFD_CONTROL_2_REGID_PATCHID__MASK 0x000000ff +#define A6XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT 0 +static inline uint32_t A6XX_VFD_CONTROL_2_REGID_PATCHID(uint32_t val) +{ + return ((val) << A6XX_VFD_CONTROL_2_REGID_PATCHID__SHIFT) & A6XX_VFD_CONTROL_2_REGID_PATCHID__MASK; +} + +#define REG_A6XX_VFD_CONTROL_3 0x0000a003 +#define A6XX_VFD_CONTROL_3_REGID_PATCHID__MASK 0x0000ff00 +#define A6XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT 8 +static inline uint32_t A6XX_VFD_CONTROL_3_REGID_PATCHID(uint32_t val) +{ + return ((val) << A6XX_VFD_CONTROL_3_REGID_PATCHID__SHIFT) & A6XX_VFD_CONTROL_3_REGID_PATCHID__MASK; +} +#define A6XX_VFD_CONTROL_3_REGID_TESSX__MASK 0x00ff0000 +#define A6XX_VFD_CONTROL_3_REGID_TESSX__SHIFT 16 +static inline uint32_t A6XX_VFD_CONTROL_3_REGID_TESSX(uint32_t val) +{ + return ((val) << A6XX_VFD_CONTROL_3_REGID_TESSX__SHIFT) & A6XX_VFD_CONTROL_3_REGID_TESSX__MASK; +} +#define A6XX_VFD_CONTROL_3_REGID_TESSY__MASK 0xff000000 +#define A6XX_VFD_CONTROL_3_REGID_TESSY__SHIFT 24 +static inline uint32_t A6XX_VFD_CONTROL_3_REGID_TESSY(uint32_t val) +{ + return ((val) << A6XX_VFD_CONTROL_3_REGID_TESSY__SHIFT) & A6XX_VFD_CONTROL_3_REGID_TESSY__MASK; +} + +#define REG_A6XX_VFD_CONTROL_4 0x0000a004 + +#define REG_A6XX_VFD_CONTROL_5 0x0000a005 + +#define REG_A6XX_VFD_CONTROL_6 0x0000a006 + +#define REG_A6XX_VFD_MODE_CNTL 0x0000a007 +#define A6XX_VFD_MODE_CNTL_BINNING_PASS 0x00000001 + +#define REG_A6XX_VFD_UNKNOWN_A008 0x0000a008 + +#define REG_A6XX_VFD_INDEX_OFFSET 0x0000a00e + +#define REG_A6XX_VFD_INSTANCE_START_OFFSET 0x0000a00f + +static inline uint32_t REG_A6XX_VFD_FETCH(uint32_t i0) { return 0x0000a010 + 0x4*i0; } + +static inline uint32_t REG_A6XX_VFD_FETCH_BASE_LO(uint32_t i0) { return 0x0000a010 + 0x4*i0; } + +static inline uint32_t REG_A6XX_VFD_FETCH_BASE_HI(uint32_t i0) { return 0x0000a011 + 0x4*i0; } + +static inline uint32_t REG_A6XX_VFD_FETCH_SIZE(uint32_t i0) { return 0x0000a012 + 0x4*i0; } + +static inline uint32_t REG_A6XX_VFD_FETCH_STRIDE(uint32_t i0) { return 0x0000a013 + 0x4*i0; } + +static inline uint32_t REG_A6XX_VFD_DECODE(uint32_t i0) { return 0x0000a090 + 0x2*i0; } + +static inline uint32_t REG_A6XX_VFD_DECODE_INSTR(uint32_t i0) { return 0x0000a090 + 0x2*i0; } +#define A6XX_VFD_DECODE_INSTR_IDX__MASK 0x0000001f +#define A6XX_VFD_DECODE_INSTR_IDX__SHIFT 0 +static inline uint32_t A6XX_VFD_DECODE_INSTR_IDX(uint32_t val) +{ + return ((val) << A6XX_VFD_DECODE_INSTR_IDX__SHIFT) & A6XX_VFD_DECODE_INSTR_IDX__MASK; +} +#define A6XX_VFD_DECODE_INSTR_INSTANCED 0x00020000 +#define A6XX_VFD_DECODE_INSTR_FORMAT__MASK 0x0ff00000 +#define A6XX_VFD_DECODE_INSTR_FORMAT__SHIFT 20 +static inline uint32_t A6XX_VFD_DECODE_INSTR_FORMAT(enum a6xx_vtx_fmt val) +{ + return ((val) << A6XX_VFD_DECODE_INSTR_FORMAT__SHIFT) & A6XX_VFD_DECODE_INSTR_FORMAT__MASK; +} +#define A6XX_VFD_DECODE_INSTR_SWAP__MASK 0x30000000 +#define A6XX_VFD_DECODE_INSTR_SWAP__SHIFT 28 +static inline uint32_t A6XX_VFD_DECODE_INSTR_SWAP(enum a3xx_color_swap val) +{ + return ((val) << A6XX_VFD_DECODE_INSTR_SWAP__SHIFT) & A6XX_VFD_DECODE_INSTR_SWAP__MASK; +} +#define A6XX_VFD_DECODE_INSTR_UNK30 0x40000000 +#define A6XX_VFD_DECODE_INSTR_FLOAT 0x80000000 + +static inline uint32_t REG_A6XX_VFD_DECODE_STEP_RATE(uint32_t i0) { return 0x0000a091 + 0x2*i0; } + +static inline uint32_t REG_A6XX_VFD_DEST_CNTL(uint32_t i0) { return 0x0000a0d0 + 0x1*i0; } + +static inline uint32_t REG_A6XX_VFD_DEST_CNTL_INSTR(uint32_t i0) { return 0x0000a0d0 + 0x1*i0; } +#define A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK 0x0000000f +#define A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT 0 +static inline uint32_t A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK(uint32_t val) +{ + return ((val) << A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__SHIFT) & A6XX_VFD_DEST_CNTL_INSTR_WRITEMASK__MASK; +} +#define A6XX_VFD_DEST_CNTL_INSTR_REGID__MASK 0x00000ff0 +#define A6XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT 4 +static inline uint32_t A6XX_VFD_DEST_CNTL_INSTR_REGID(uint32_t val) +{ + return ((val) << A6XX_VFD_DEST_CNTL_INSTR_REGID__SHIFT) & A6XX_VFD_DEST_CNTL_INSTR_REGID__MASK; +} + +#define REG_A6XX_SP_UNKNOWN_A0F8 0x0000a0f8 + +#define REG_A6XX_SP_PRIMITIVE_CNTL 0x0000a802 +#define A6XX_SP_PRIMITIVE_CNTL_VSOUT__MASK 0x0000001f +#define A6XX_SP_PRIMITIVE_CNTL_VSOUT__SHIFT 0 +static inline uint32_t A6XX_SP_PRIMITIVE_CNTL_VSOUT(uint32_t val) +{ + return ((val) << A6XX_SP_PRIMITIVE_CNTL_VSOUT__SHIFT) & A6XX_SP_PRIMITIVE_CNTL_VSOUT__MASK; +} + +static inline uint32_t REG_A6XX_SP_VS_OUT(uint32_t i0) { return 0x0000a803 + 0x1*i0; } + +static inline uint32_t REG_A6XX_SP_VS_OUT_REG(uint32_t i0) { return 0x0000a803 + 0x1*i0; } +#define A6XX_SP_VS_OUT_REG_A_REGID__MASK 0x000000ff +#define A6XX_SP_VS_OUT_REG_A_REGID__SHIFT 0 +static inline uint32_t A6XX_SP_VS_OUT_REG_A_REGID(uint32_t val) +{ + return ((val) << A6XX_SP_VS_OUT_REG_A_REGID__SHIFT) & A6XX_SP_VS_OUT_REG_A_REGID__MASK; +} +#define A6XX_SP_VS_OUT_REG_A_COMPMASK__MASK 0x00000f00 +#define A6XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT 8 +static inline uint32_t A6XX_SP_VS_OUT_REG_A_COMPMASK(uint32_t val) +{ + return ((val) << A6XX_SP_VS_OUT_REG_A_COMPMASK__SHIFT) & A6XX_SP_VS_OUT_REG_A_COMPMASK__MASK; +} +#define A6XX_SP_VS_OUT_REG_B_REGID__MASK 0x00ff0000 +#define A6XX_SP_VS_OUT_REG_B_REGID__SHIFT 16 +static inline uint32_t A6XX_SP_VS_OUT_REG_B_REGID(uint32_t val) +{ + return ((val) << A6XX_SP_VS_OUT_REG_B_REGID__SHIFT) & A6XX_SP_VS_OUT_REG_B_REGID__MASK; +} +#define A6XX_SP_VS_OUT_REG_B_COMPMASK__MASK 0x0f000000 +#define A6XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT 24 +static inline uint32_t A6XX_SP_VS_OUT_REG_B_COMPMASK(uint32_t val) +{ + return ((val) << A6XX_SP_VS_OUT_REG_B_COMPMASK__SHIFT) & A6XX_SP_VS_OUT_REG_B_COMPMASK__MASK; +} + +static inline uint32_t REG_A6XX_SP_VS_VPC_DST(uint32_t i0) { return 0x0000a813 + 0x1*i0; } + +static inline uint32_t REG_A6XX_SP_VS_VPC_DST_REG(uint32_t i0) { return 0x0000a813 + 0x1*i0; } +#define A6XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK 0x000000ff +#define A6XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT 0 +static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC0(uint32_t val) +{ + return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC0__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC0__MASK; +} +#define A6XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK 0x0000ff00 +#define A6XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT 8 +static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC1(uint32_t val) +{ + return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC1__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC1__MASK; +} +#define A6XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK 0x00ff0000 +#define A6XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT 16 +static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC2(uint32_t val) +{ + return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC2__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC2__MASK; +} +#define A6XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK 0xff000000 +#define A6XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT 24 +static inline uint32_t A6XX_SP_VS_VPC_DST_REG_OUTLOC3(uint32_t val) +{ + return ((val) << A6XX_SP_VS_VPC_DST_REG_OUTLOC3__SHIFT) & A6XX_SP_VS_VPC_DST_REG_OUTLOC3__MASK; +} + +#define REG_A6XX_SP_VS_CTRL_REG0 0x0000a800 +#define A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x0000007e +#define A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 1 +static inline uint32_t A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_VS_CTRL_REG0_HALFREGFOOTPRINT__MASK; +} +#define A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x00001f80 +#define A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 7 +static inline uint32_t A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_VS_CTRL_REG0_FULLREGFOOTPRINT__MASK; +} +#define A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__MASK 0x000fc000 +#define A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__SHIFT 14 +static inline uint32_t A6XX_SP_VS_CTRL_REG0_BRANCHSTACK(uint32_t val) +{ + return ((val) << A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_VS_CTRL_REG0_BRANCHSTACK__MASK; +} +#define A6XX_SP_VS_CTRL_REG0_THREADSIZE__MASK 0x00100000 +#define A6XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT 20 +static inline uint32_t A6XX_SP_VS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) +{ + return ((val) << A6XX_SP_VS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_VS_CTRL_REG0_THREADSIZE__MASK; +} +#define A6XX_SP_VS_CTRL_REG0_VARYING 0x00400000 +#define A6XX_SP_VS_CTRL_REG0_PIXLODENABLE 0x04000000 +#define A6XX_SP_VS_CTRL_REG0_MERGEDREGS 0x80000000 + +#define REG_A6XX_SP_VS_OBJ_START_LO 0x0000a81c + +#define REG_A6XX_SP_VS_OBJ_START_HI 0x0000a81d + +#define REG_A6XX_SP_VS_TEX_COUNT 0x0000a822 + +#define REG_A6XX_SP_VS_CONFIG 0x0000a823 +#define A6XX_SP_VS_CONFIG_ENABLED 0x00000100 +#define A6XX_SP_VS_CONFIG_NTEX__MASK 0x0001fe00 +#define A6XX_SP_VS_CONFIG_NTEX__SHIFT 9 +static inline uint32_t A6XX_SP_VS_CONFIG_NTEX(uint32_t val) +{ + return ((val) << A6XX_SP_VS_CONFIG_NTEX__SHIFT) & A6XX_SP_VS_CONFIG_NTEX__MASK; +} +#define A6XX_SP_VS_CONFIG_NSAMP__MASK 0x01fe0000 +#define A6XX_SP_VS_CONFIG_NSAMP__SHIFT 17 +static inline uint32_t A6XX_SP_VS_CONFIG_NSAMP(uint32_t val) +{ + return ((val) << A6XX_SP_VS_CONFIG_NSAMP__SHIFT) & A6XX_SP_VS_CONFIG_NSAMP__MASK; +} + +#define REG_A6XX_SP_VS_INSTRLEN 0x0000a824 + +#define REG_A6XX_SP_HS_CTRL_REG0 0x0000a830 +#define A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x0000007e +#define A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 1 +static inline uint32_t A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_HS_CTRL_REG0_HALFREGFOOTPRINT__MASK; +} +#define A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x00001f80 +#define A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 7 +static inline uint32_t A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_HS_CTRL_REG0_FULLREGFOOTPRINT__MASK; +} +#define A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK 0x000fc000 +#define A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT 14 +static inline uint32_t A6XX_SP_HS_CTRL_REG0_BRANCHSTACK(uint32_t val) +{ + return ((val) << A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_HS_CTRL_REG0_BRANCHSTACK__MASK; +} +#define A6XX_SP_HS_CTRL_REG0_THREADSIZE__MASK 0x00100000 +#define A6XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT 20 +static inline uint32_t A6XX_SP_HS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) +{ + return ((val) << A6XX_SP_HS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_HS_CTRL_REG0_THREADSIZE__MASK; +} +#define A6XX_SP_HS_CTRL_REG0_VARYING 0x00400000 +#define A6XX_SP_HS_CTRL_REG0_PIXLODENABLE 0x04000000 +#define A6XX_SP_HS_CTRL_REG0_MERGEDREGS 0x80000000 + +#define REG_A6XX_SP_HS_UNKNOWN_A831 0x0000a831 + +#define REG_A6XX_SP_HS_OBJ_START_LO 0x0000a834 + +#define REG_A6XX_SP_HS_OBJ_START_HI 0x0000a835 + +#define REG_A6XX_SP_HS_TEX_COUNT 0x0000a83a + +#define REG_A6XX_SP_HS_CONFIG 0x0000a83b +#define A6XX_SP_HS_CONFIG_ENABLED 0x00000100 +#define A6XX_SP_HS_CONFIG_NTEX__MASK 0x0001fe00 +#define A6XX_SP_HS_CONFIG_NTEX__SHIFT 9 +static inline uint32_t A6XX_SP_HS_CONFIG_NTEX(uint32_t val) +{ + return ((val) << A6XX_SP_HS_CONFIG_NTEX__SHIFT) & A6XX_SP_HS_CONFIG_NTEX__MASK; +} +#define A6XX_SP_HS_CONFIG_NSAMP__MASK 0x01fe0000 +#define A6XX_SP_HS_CONFIG_NSAMP__SHIFT 17 +static inline uint32_t A6XX_SP_HS_CONFIG_NSAMP(uint32_t val) +{ + return ((val) << A6XX_SP_HS_CONFIG_NSAMP__SHIFT) & A6XX_SP_HS_CONFIG_NSAMP__MASK; +} + +#define REG_A6XX_SP_HS_INSTRLEN 0x0000a83c + +#define REG_A6XX_SP_DS_CTRL_REG0 0x0000a840 +#define A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x0000007e +#define A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 1 +static inline uint32_t A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_DS_CTRL_REG0_HALFREGFOOTPRINT__MASK; +} +#define A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x00001f80 +#define A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 7 +static inline uint32_t A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_DS_CTRL_REG0_FULLREGFOOTPRINT__MASK; +} +#define A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK 0x000fc000 +#define A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT 14 +static inline uint32_t A6XX_SP_DS_CTRL_REG0_BRANCHSTACK(uint32_t val) +{ + return ((val) << A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_DS_CTRL_REG0_BRANCHSTACK__MASK; +} +#define A6XX_SP_DS_CTRL_REG0_THREADSIZE__MASK 0x00100000 +#define A6XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT 20 +static inline uint32_t A6XX_SP_DS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) +{ + return ((val) << A6XX_SP_DS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_DS_CTRL_REG0_THREADSIZE__MASK; +} +#define A6XX_SP_DS_CTRL_REG0_VARYING 0x00400000 +#define A6XX_SP_DS_CTRL_REG0_PIXLODENABLE 0x04000000 +#define A6XX_SP_DS_CTRL_REG0_MERGEDREGS 0x80000000 + +#define REG_A6XX_SP_DS_OBJ_START_LO 0x0000a85c + +#define REG_A6XX_SP_DS_OBJ_START_HI 0x0000a85d + +#define REG_A6XX_SP_DS_TEX_COUNT 0x0000a862 + +#define REG_A6XX_SP_DS_CONFIG 0x0000a863 +#define A6XX_SP_DS_CONFIG_ENABLED 0x00000100 +#define A6XX_SP_DS_CONFIG_NTEX__MASK 0x0001fe00 +#define A6XX_SP_DS_CONFIG_NTEX__SHIFT 9 +static inline uint32_t A6XX_SP_DS_CONFIG_NTEX(uint32_t val) +{ + return ((val) << A6XX_SP_DS_CONFIG_NTEX__SHIFT) & A6XX_SP_DS_CONFIG_NTEX__MASK; +} +#define A6XX_SP_DS_CONFIG_NSAMP__MASK 0x01fe0000 +#define A6XX_SP_DS_CONFIG_NSAMP__SHIFT 17 +static inline uint32_t A6XX_SP_DS_CONFIG_NSAMP(uint32_t val) +{ + return ((val) << A6XX_SP_DS_CONFIG_NSAMP__SHIFT) & A6XX_SP_DS_CONFIG_NSAMP__MASK; +} + +#define REG_A6XX_SP_DS_INSTRLEN 0x0000a864 + +#define REG_A6XX_SP_GS_CTRL_REG0 0x0000a870 +#define A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x0000007e +#define A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 1 +static inline uint32_t A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_GS_CTRL_REG0_HALFREGFOOTPRINT__MASK; +} +#define A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x00001f80 +#define A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 7 +static inline uint32_t A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_GS_CTRL_REG0_FULLREGFOOTPRINT__MASK; +} +#define A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK 0x000fc000 +#define A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT 14 +static inline uint32_t A6XX_SP_GS_CTRL_REG0_BRANCHSTACK(uint32_t val) +{ + return ((val) << A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_GS_CTRL_REG0_BRANCHSTACK__MASK; +} +#define A6XX_SP_GS_CTRL_REG0_THREADSIZE__MASK 0x00100000 +#define A6XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT 20 +static inline uint32_t A6XX_SP_GS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) +{ + return ((val) << A6XX_SP_GS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_GS_CTRL_REG0_THREADSIZE__MASK; +} +#define A6XX_SP_GS_CTRL_REG0_VARYING 0x00400000 +#define A6XX_SP_GS_CTRL_REG0_PIXLODENABLE 0x04000000 +#define A6XX_SP_GS_CTRL_REG0_MERGEDREGS 0x80000000 + +#define REG_A6XX_SP_GS_UNKNOWN_A871 0x0000a871 + +#define REG_A6XX_SP_GS_OBJ_START_LO 0x0000a88d + +#define REG_A6XX_SP_GS_OBJ_START_HI 0x0000a88e + +#define REG_A6XX_SP_GS_TEX_COUNT 0x0000a893 + +#define REG_A6XX_SP_GS_CONFIG 0x0000a894 +#define A6XX_SP_GS_CONFIG_ENABLED 0x00000100 +#define A6XX_SP_GS_CONFIG_NTEX__MASK 0x0001fe00 +#define A6XX_SP_GS_CONFIG_NTEX__SHIFT 9 +static inline uint32_t A6XX_SP_GS_CONFIG_NTEX(uint32_t val) +{ + return ((val) << A6XX_SP_GS_CONFIG_NTEX__SHIFT) & A6XX_SP_GS_CONFIG_NTEX__MASK; +} +#define A6XX_SP_GS_CONFIG_NSAMP__MASK 0x01fe0000 +#define A6XX_SP_GS_CONFIG_NSAMP__SHIFT 17 +static inline uint32_t A6XX_SP_GS_CONFIG_NSAMP(uint32_t val) +{ + return ((val) << A6XX_SP_GS_CONFIG_NSAMP__SHIFT) & A6XX_SP_GS_CONFIG_NSAMP__MASK; +} + +#define REG_A6XX_SP_GS_INSTRLEN 0x0000a895 + +#define REG_A6XX_SP_VS_TEX_SAMP_LO 0x0000a8a0 + +#define REG_A6XX_SP_VS_TEX_SAMP_HI 0x0000a8a1 + +#define REG_A6XX_SP_HS_TEX_SAMP_LO 0x0000a8a2 + +#define REG_A6XX_SP_HS_TEX_SAMP_HI 0x0000a8a3 + +#define REG_A6XX_SP_DS_TEX_SAMP_LO 0x0000a8a4 + +#define REG_A6XX_SP_DS_TEX_SAMP_HI 0x0000a8a5 + +#define REG_A6XX_SP_GS_TEX_SAMP_LO 0x0000a8a6 + +#define REG_A6XX_SP_GS_TEX_SAMP_HI 0x0000a8a7 + +#define REG_A6XX_SP_VS_TEX_CONST_LO 0x0000a8a8 + +#define REG_A6XX_SP_VS_TEX_CONST_HI 0x0000a8a9 + +#define REG_A6XX_SP_HS_TEX_CONST_LO 0x0000a8aa + +#define REG_A6XX_SP_HS_TEX_CONST_HI 0x0000a8ab + +#define REG_A6XX_SP_DS_TEX_CONST_LO 0x0000a8ac + +#define REG_A6XX_SP_DS_TEX_CONST_HI 0x0000a8ad + +#define REG_A6XX_SP_GS_TEX_CONST_LO 0x0000a8ae + +#define REG_A6XX_SP_GS_TEX_CONST_HI 0x0000a8af + +#define REG_A6XX_SP_FS_CTRL_REG0 0x0000a980 +#define A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x0000007e +#define A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 1 +static inline uint32_t A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_FS_CTRL_REG0_HALFREGFOOTPRINT__MASK; +} +#define A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x00001f80 +#define A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 7 +static inline uint32_t A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_FS_CTRL_REG0_FULLREGFOOTPRINT__MASK; +} +#define A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__MASK 0x000fc000 +#define A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__SHIFT 14 +static inline uint32_t A6XX_SP_FS_CTRL_REG0_BRANCHSTACK(uint32_t val) +{ + return ((val) << A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_FS_CTRL_REG0_BRANCHSTACK__MASK; +} +#define A6XX_SP_FS_CTRL_REG0_THREADSIZE__MASK 0x00100000 +#define A6XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT 20 +static inline uint32_t A6XX_SP_FS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) +{ + return ((val) << A6XX_SP_FS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_FS_CTRL_REG0_THREADSIZE__MASK; +} +#define A6XX_SP_FS_CTRL_REG0_VARYING 0x00400000 +#define A6XX_SP_FS_CTRL_REG0_PIXLODENABLE 0x04000000 +#define A6XX_SP_FS_CTRL_REG0_MERGEDREGS 0x80000000 + +#define REG_A6XX_SP_FS_OBJ_START_LO 0x0000a983 + +#define REG_A6XX_SP_FS_OBJ_START_HI 0x0000a984 + +#define REG_A6XX_SP_BLEND_CNTL 0x0000a989 +#define A6XX_SP_BLEND_CNTL_ENABLED 0x00000001 +#define A6XX_SP_BLEND_CNTL_UNK8 0x00000100 + +#define REG_A6XX_SP_SRGB_CNTL 0x0000a98a +#define A6XX_SP_SRGB_CNTL_SRGB_MRT0 0x00000001 +#define A6XX_SP_SRGB_CNTL_SRGB_MRT1 0x00000002 +#define A6XX_SP_SRGB_CNTL_SRGB_MRT2 0x00000004 +#define A6XX_SP_SRGB_CNTL_SRGB_MRT3 0x00000008 +#define A6XX_SP_SRGB_CNTL_SRGB_MRT4 0x00000010 +#define A6XX_SP_SRGB_CNTL_SRGB_MRT5 0x00000020 +#define A6XX_SP_SRGB_CNTL_SRGB_MRT6 0x00000040 +#define A6XX_SP_SRGB_CNTL_SRGB_MRT7 0x00000080 + +#define REG_A6XX_SP_FS_RENDER_COMPONENTS 0x0000a98b +#define A6XX_SP_FS_RENDER_COMPONENTS_RT0__MASK 0x0000000f +#define A6XX_SP_FS_RENDER_COMPONENTS_RT0__SHIFT 0 +static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT0(uint32_t val) +{ + return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT0__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT0__MASK; +} +#define A6XX_SP_FS_RENDER_COMPONENTS_RT1__MASK 0x000000f0 +#define A6XX_SP_FS_RENDER_COMPONENTS_RT1__SHIFT 4 +static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT1(uint32_t val) +{ + return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT1__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT1__MASK; +} +#define A6XX_SP_FS_RENDER_COMPONENTS_RT2__MASK 0x00000f00 +#define A6XX_SP_FS_RENDER_COMPONENTS_RT2__SHIFT 8 +static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT2(uint32_t val) +{ + return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT2__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT2__MASK; +} +#define A6XX_SP_FS_RENDER_COMPONENTS_RT3__MASK 0x0000f000 +#define A6XX_SP_FS_RENDER_COMPONENTS_RT3__SHIFT 12 +static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT3(uint32_t val) +{ + return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT3__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT3__MASK; +} +#define A6XX_SP_FS_RENDER_COMPONENTS_RT4__MASK 0x000f0000 +#define A6XX_SP_FS_RENDER_COMPONENTS_RT4__SHIFT 16 +static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT4(uint32_t val) +{ + return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT4__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT4__MASK; +} +#define A6XX_SP_FS_RENDER_COMPONENTS_RT5__MASK 0x00f00000 +#define A6XX_SP_FS_RENDER_COMPONENTS_RT5__SHIFT 20 +static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT5(uint32_t val) +{ + return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT5__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT5__MASK; +} +#define A6XX_SP_FS_RENDER_COMPONENTS_RT6__MASK 0x0f000000 +#define A6XX_SP_FS_RENDER_COMPONENTS_RT6__SHIFT 24 +static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT6(uint32_t val) +{ + return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT6__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT6__MASK; +} +#define A6XX_SP_FS_RENDER_COMPONENTS_RT7__MASK 0xf0000000 +#define A6XX_SP_FS_RENDER_COMPONENTS_RT7__SHIFT 28 +static inline uint32_t A6XX_SP_FS_RENDER_COMPONENTS_RT7(uint32_t val) +{ + return ((val) << A6XX_SP_FS_RENDER_COMPONENTS_RT7__SHIFT) & A6XX_SP_FS_RENDER_COMPONENTS_RT7__MASK; +} + +#define REG_A6XX_SP_FS_OUTPUT_CNTL0 0x0000a98c +#define A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__MASK 0x0000ff00 +#define A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__SHIFT 8 +static inline uint32_t A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID(uint32_t val) +{ + return ((val) << A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__SHIFT) & A6XX_SP_FS_OUTPUT_CNTL0_DEPTH_REGID__MASK; +} + +#define REG_A6XX_SP_FS_OUTPUT_CNTL1 0x0000a98d +#define A6XX_SP_FS_OUTPUT_CNTL1_MRT__MASK 0x0000000f +#define A6XX_SP_FS_OUTPUT_CNTL1_MRT__SHIFT 0 +static inline uint32_t A6XX_SP_FS_OUTPUT_CNTL1_MRT(uint32_t val) +{ + return ((val) << A6XX_SP_FS_OUTPUT_CNTL1_MRT__SHIFT) & A6XX_SP_FS_OUTPUT_CNTL1_MRT__MASK; +} + +static inline uint32_t REG_A6XX_SP_FS_MRT(uint32_t i0) { return 0x0000a996 + 0x1*i0; } + +static inline uint32_t REG_A6XX_SP_FS_MRT_REG(uint32_t i0) { return 0x0000a996 + 0x1*i0; } +#define A6XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK 0x000000ff +#define A6XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT 0 +static inline uint32_t A6XX_SP_FS_MRT_REG_COLOR_FORMAT(enum a6xx_color_fmt val) +{ + return ((val) << A6XX_SP_FS_MRT_REG_COLOR_FORMAT__SHIFT) & A6XX_SP_FS_MRT_REG_COLOR_FORMAT__MASK; +} +#define A6XX_SP_FS_MRT_REG_COLOR_SINT 0x00000100 +#define A6XX_SP_FS_MRT_REG_COLOR_UINT 0x00000200 +#define A6XX_SP_FS_MRT_REG_COLOR_SRGB 0x00000400 + +#define REG_A6XX_SP_FS_TEX_COUNT 0x0000a9a7 + +#define REG_A6XX_SP_UNKNOWN_A9A8 0x0000a9a8 + +#define REG_A6XX_SP_FS_TEX_SAMP_LO 0x0000a9e0 + +#define REG_A6XX_SP_FS_TEX_SAMP_HI 0x0000a9e1 + +#define REG_A6XX_SP_CS_TEX_SAMP_LO 0x0000a9e2 + +#define REG_A6XX_SP_CS_TEX_SAMP_HI 0x0000a9e3 + +#define REG_A6XX_SP_FS_TEX_CONST_LO 0x0000a9e4 + +#define REG_A6XX_SP_FS_TEX_CONST_HI 0x0000a9e5 + +#define REG_A6XX_SP_CS_TEX_CONST_LO 0x0000a9e6 + +#define REG_A6XX_SP_CS_TEX_CONST_HI 0x0000a9e7 + +static inline uint32_t REG_A6XX_SP_FS_OUTPUT(uint32_t i0) { return 0x0000a98e + 0x1*i0; } + +static inline uint32_t REG_A6XX_SP_FS_OUTPUT_REG(uint32_t i0) { return 0x0000a98e + 0x1*i0; } +#define A6XX_SP_FS_OUTPUT_REG_REGID__MASK 0x000000ff +#define A6XX_SP_FS_OUTPUT_REG_REGID__SHIFT 0 +static inline uint32_t A6XX_SP_FS_OUTPUT_REG_REGID(uint32_t val) +{ + return ((val) << A6XX_SP_FS_OUTPUT_REG_REGID__SHIFT) & A6XX_SP_FS_OUTPUT_REG_REGID__MASK; +} +#define A6XX_SP_FS_OUTPUT_REG_HALF_PRECISION 0x00000100 + +#define REG_A6XX_SP_CS_CTRL_REG0 0x0000a9b0 +#define A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK 0x0000007e +#define A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT 1 +static inline uint32_t A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__SHIFT) & A6XX_SP_CS_CTRL_REG0_HALFREGFOOTPRINT__MASK; +} +#define A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK 0x00001f80 +#define A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT 7 +static inline uint32_t A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT(uint32_t val) +{ + return ((val) << A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__SHIFT) & A6XX_SP_CS_CTRL_REG0_FULLREGFOOTPRINT__MASK; +} +#define A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK 0x000fc000 +#define A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT 14 +static inline uint32_t A6XX_SP_CS_CTRL_REG0_BRANCHSTACK(uint32_t val) +{ + return ((val) << A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__SHIFT) & A6XX_SP_CS_CTRL_REG0_BRANCHSTACK__MASK; +} +#define A6XX_SP_CS_CTRL_REG0_THREADSIZE__MASK 0x00100000 +#define A6XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT 20 +static inline uint32_t A6XX_SP_CS_CTRL_REG0_THREADSIZE(enum a3xx_threadsize val) +{ + return ((val) << A6XX_SP_CS_CTRL_REG0_THREADSIZE__SHIFT) & A6XX_SP_CS_CTRL_REG0_THREADSIZE__MASK; +} +#define A6XX_SP_CS_CTRL_REG0_VARYING 0x00400000 +#define A6XX_SP_CS_CTRL_REG0_PIXLODENABLE 0x04000000 +#define A6XX_SP_CS_CTRL_REG0_MERGEDREGS 0x80000000 + +#define REG_A6XX_SP_CS_OBJ_START_LO 0x0000a9b4 + +#define REG_A6XX_SP_CS_OBJ_START_HI 0x0000a9b5 + +#define REG_A6XX_SP_CS_INSTRLEN 0x0000a9bc + +#define REG_A6XX_SP_UNKNOWN_AB00 0x0000ab00 + +#define REG_A6XX_SP_FS_CONFIG 0x0000ab04 +#define A6XX_SP_FS_CONFIG_ENABLED 0x00000100 +#define A6XX_SP_FS_CONFIG_NTEX__MASK 0x0001fe00 +#define A6XX_SP_FS_CONFIG_NTEX__SHIFT 9 +static inline uint32_t A6XX_SP_FS_CONFIG_NTEX(uint32_t val) +{ + return ((val) << A6XX_SP_FS_CONFIG_NTEX__SHIFT) & A6XX_SP_FS_CONFIG_NTEX__MASK; +} +#define A6XX_SP_FS_CONFIG_NSAMP__MASK 0x01fe0000 +#define A6XX_SP_FS_CONFIG_NSAMP__SHIFT 17 +static inline uint32_t A6XX_SP_FS_CONFIG_NSAMP(uint32_t val) +{ + return ((val) << A6XX_SP_FS_CONFIG_NSAMP__SHIFT) & A6XX_SP_FS_CONFIG_NSAMP__MASK; +} + +#define REG_A6XX_SP_FS_INSTRLEN 0x0000ab05 + +#define REG_A6XX_SP_UNKNOWN_AE00 0x0000ae00 + +#define REG_A6XX_SP_UNKNOWN_AE04 0x0000ae04 + +#define REG_A6XX_SP_UNKNOWN_AE0F 0x0000ae0f + +#define REG_A6XX_SP_UNKNOWN_B182 0x0000b182 + +#define REG_A6XX_SP_TP_RAS_MSAA_CNTL 0x0000b300 +#define A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__MASK 0x00000003 +#define A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT 0 +static inline uint32_t A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val) +{ + return ((val) << A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_SP_TP_RAS_MSAA_CNTL_SAMPLES__MASK; +} + +#define REG_A6XX_SP_TP_DEST_MSAA_CNTL 0x0000b301 +#define A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__MASK 0x00000003 +#define A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT 0 +static inline uint32_t A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES(enum a3xx_msaa_samples val) +{ + return ((val) << A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__SHIFT) & A6XX_SP_TP_DEST_MSAA_CNTL_SAMPLES__MASK; +} +#define A6XX_SP_TP_DEST_MSAA_CNTL_MSAA_DISABLE 0x00000004 + +#define REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_LO 0x0000b302 + +#define REG_A6XX_SP_TP_BORDER_COLOR_BASE_ADDR_HI 0x0000b303 + +#define REG_A6XX_SP_TP_UNKNOWN_B304 0x0000b304 + +#define REG_A6XX_SP_PS_2D_SRC_INFO 0x0000b4c0 +#define A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK 0x000000ff +#define A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__SHIFT 0 +static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT(enum a6xx_color_fmt val) +{ + return ((val) << A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_COLOR_FORMAT__MASK; +} +#define A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__MASK 0x00000300 +#define A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__SHIFT 8 +static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_TILE_MODE(enum a6xx_tile_mode val) +{ + return ((val) << A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_TILE_MODE__MASK; +} +#define A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__MASK 0x00000c00 +#define A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__SHIFT 10 +static inline uint32_t A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP(enum a3xx_color_swap val) +{ + return ((val) << A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__SHIFT) & A6XX_SP_PS_2D_SRC_INFO_COLOR_SWAP__MASK; +} +#define A6XX_SP_PS_2D_SRC_INFO_FLAGS 0x00001000 + +#define REG_A6XX_SP_PS_2D_SRC_LO 0x0000b4c2 + +#define REG_A6XX_SP_PS_2D_SRC_HI 0x0000b4c3 + +#define REG_A6XX_SP_PS_2D_SRC_FLAGS_LO 0x0000b4ca + +#define REG_A6XX_SP_PS_2D_SRC_FLAGS_HI 0x0000b4cb + +#define REG_A6XX_SP_UNKNOWN_B600 0x0000b600 + +#define REG_A6XX_SP_UNKNOWN_B605 0x0000b605 + +#define REG_A6XX_HLSQ_VS_CNTL 0x0000b800 +#define A6XX_HLSQ_VS_CNTL_CONSTLEN__MASK 0x000000ff +#define A6XX_HLSQ_VS_CNTL_CONSTLEN__SHIFT 0 +static inline uint32_t A6XX_HLSQ_VS_CNTL_CONSTLEN(uint32_t val) +{ + return ((val >> 2) << A6XX_HLSQ_VS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_VS_CNTL_CONSTLEN__MASK; +} + +#define REG_A6XX_HLSQ_HS_CNTL 0x0000b801 +#define A6XX_HLSQ_HS_CNTL_CONSTLEN__MASK 0x000000ff +#define A6XX_HLSQ_HS_CNTL_CONSTLEN__SHIFT 0 +static inline uint32_t A6XX_HLSQ_HS_CNTL_CONSTLEN(uint32_t val) +{ + return ((val >> 2) << A6XX_HLSQ_HS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_HS_CNTL_CONSTLEN__MASK; +} + +#define REG_A6XX_HLSQ_DS_CNTL 0x0000b802 +#define A6XX_HLSQ_DS_CNTL_CONSTLEN__MASK 0x000000ff +#define A6XX_HLSQ_DS_CNTL_CONSTLEN__SHIFT 0 +static inline uint32_t A6XX_HLSQ_DS_CNTL_CONSTLEN(uint32_t val) +{ + return ((val >> 2) << A6XX_HLSQ_DS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_DS_CNTL_CONSTLEN__MASK; +} + +#define REG_A6XX_HLSQ_GS_CNTL 0x0000b803 +#define A6XX_HLSQ_GS_CNTL_CONSTLEN__MASK 0x000000ff +#define A6XX_HLSQ_GS_CNTL_CONSTLEN__SHIFT 0 +static inline uint32_t A6XX_HLSQ_GS_CNTL_CONSTLEN(uint32_t val) +{ + return ((val >> 2) << A6XX_HLSQ_GS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_GS_CNTL_CONSTLEN__MASK; +} + +#define REG_A6XX_HLSQ_CONTROL_1_REG 0x0000b982 + +#define REG_A6XX_HLSQ_CONTROL_2_REG 0x0000b983 +#define A6XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK 0x000000ff +#define A6XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_FACEREGID(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CONTROL_2_REG_FACEREGID__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_FACEREGID__MASK; +} +#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK 0x0000ff00 +#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT 8 +static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_SAMPLEID(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_SAMPLEID__MASK; +} +#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK 0x00ff0000 +#define A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT 16 +static inline uint32_t A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__SHIFT) & A6XX_HLSQ_CONTROL_2_REG_SAMPLEMASK__MASK; +} + +#define REG_A6XX_HLSQ_CONTROL_3_REG 0x0000b984 +#define A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__MASK 0x000000ff +#define A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__SHIFT) & A6XX_HLSQ_CONTROL_3_REG_FRAGCOORDXYREGID__MASK; +} + +#define REG_A6XX_HLSQ_CONTROL_4_REG 0x0000b985 +#define A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK 0x00ff0000 +#define A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT 16 +static inline uint32_t A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__SHIFT) & A6XX_HLSQ_CONTROL_4_REG_XYCOORDREGID__MASK; +} +#define A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK 0xff000000 +#define A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT 24 +static inline uint32_t A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__SHIFT) & A6XX_HLSQ_CONTROL_4_REG_ZWCOORDREGID__MASK; +} + +#define REG_A6XX_HLSQ_CONTROL_5_REG 0x0000b986 + +#define REG_A6XX_HLSQ_CS_NDRANGE_0 0x0000b990 +#define A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK 0x00000003 +#define A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_KERNELDIM__MASK; +} +#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK 0x00000ffc +#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT 2 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEX__MASK; +} +#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK 0x003ff000 +#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT 12 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEY__MASK; +} +#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK 0xffc00000 +#define A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT 22 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__SHIFT) & A6XX_HLSQ_CS_NDRANGE_0_LOCALSIZEZ__MASK; +} + +#define REG_A6XX_HLSQ_CS_NDRANGE_1 0x0000b991 +#define A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK 0xffffffff +#define A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__SHIFT) & A6XX_HLSQ_CS_NDRANGE_1_GLOBALSIZE_X__MASK; +} + +#define REG_A6XX_HLSQ_CS_NDRANGE_2 0x0000b992 +#define A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK 0xffffffff +#define A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__SHIFT) & A6XX_HLSQ_CS_NDRANGE_2_GLOBALOFF_X__MASK; +} + +#define REG_A6XX_HLSQ_CS_NDRANGE_3 0x0000b993 +#define A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK 0xffffffff +#define A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__SHIFT) & A6XX_HLSQ_CS_NDRANGE_3_GLOBALSIZE_Y__MASK; +} + +#define REG_A6XX_HLSQ_CS_NDRANGE_4 0x0000b994 +#define A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK 0xffffffff +#define A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__SHIFT) & A6XX_HLSQ_CS_NDRANGE_4_GLOBALOFF_Y__MASK; +} + +#define REG_A6XX_HLSQ_CS_NDRANGE_5 0x0000b995 +#define A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK 0xffffffff +#define A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__SHIFT) & A6XX_HLSQ_CS_NDRANGE_5_GLOBALSIZE_Z__MASK; +} + +#define REG_A6XX_HLSQ_CS_NDRANGE_6 0x0000b996 +#define A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK 0xffffffff +#define A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__SHIFT) & A6XX_HLSQ_CS_NDRANGE_6_GLOBALOFF_Z__MASK; +} + +#define REG_A6XX_HLSQ_CS_CNTL_0 0x0000b997 +#define A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK 0x000000ff +#define A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__SHIFT 0 +static inline uint32_t A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__SHIFT) & A6XX_HLSQ_CS_CNTL_0_WGIDCONSTID__MASK; +} +#define A6XX_HLSQ_CS_CNTL_0_UNK0__MASK 0x0000ff00 +#define A6XX_HLSQ_CS_CNTL_0_UNK0__SHIFT 8 +static inline uint32_t A6XX_HLSQ_CS_CNTL_0_UNK0(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_CNTL_0_UNK0__SHIFT) & A6XX_HLSQ_CS_CNTL_0_UNK0__MASK; +} +#define A6XX_HLSQ_CS_CNTL_0_UNK1__MASK 0x00ff0000 +#define A6XX_HLSQ_CS_CNTL_0_UNK1__SHIFT 16 +static inline uint32_t A6XX_HLSQ_CS_CNTL_0_UNK1(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_CNTL_0_UNK1__SHIFT) & A6XX_HLSQ_CS_CNTL_0_UNK1__MASK; +} +#define A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__MASK 0xff000000 +#define A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__SHIFT 24 +static inline uint32_t A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID(uint32_t val) +{ + return ((val) << A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__SHIFT) & A6XX_HLSQ_CS_CNTL_0_LOCALIDREGID__MASK; +} + +#define REG_A6XX_HLSQ_CS_KERNEL_GROUP_X 0x0000b999 + +#define REG_A6XX_HLSQ_CS_KERNEL_GROUP_Y 0x0000b99a + +#define REG_A6XX_HLSQ_CS_KERNEL_GROUP_Z 0x0000b99b + +#define REG_A6XX_HLSQ_UPDATE_CNTL 0x0000bb08 + +#define REG_A6XX_HLSQ_FS_CNTL 0x0000bb10 +#define A6XX_HLSQ_FS_CNTL_CONSTLEN__MASK 0x000000ff +#define A6XX_HLSQ_FS_CNTL_CONSTLEN__SHIFT 0 +static inline uint32_t A6XX_HLSQ_FS_CNTL_CONSTLEN(uint32_t val) +{ + return ((val >> 2) << A6XX_HLSQ_FS_CNTL_CONSTLEN__SHIFT) & A6XX_HLSQ_FS_CNTL_CONSTLEN__MASK; +} + +#define REG_A6XX_HLSQ_UNKNOWN_BB11 0x0000bb11 + +#define REG_A6XX_HLSQ_UNKNOWN_BE00 0x0000be00 + +#define REG_A6XX_HLSQ_UNKNOWN_BE01 0x0000be01 + +#define REG_A6XX_HLSQ_UNKNOWN_BE04 0x0000be04 + +#define REG_A6XX_TEX_SAMP_0 0x00000000 +#define A6XX_TEX_SAMP_0_MIPFILTER_LINEAR_NEAR 0x00000001 +#define A6XX_TEX_SAMP_0_XY_MAG__MASK 0x00000006 +#define A6XX_TEX_SAMP_0_XY_MAG__SHIFT 1 +static inline uint32_t A6XX_TEX_SAMP_0_XY_MAG(enum a6xx_tex_filter val) +{ + return ((val) << A6XX_TEX_SAMP_0_XY_MAG__SHIFT) & A6XX_TEX_SAMP_0_XY_MAG__MASK; +} +#define A6XX_TEX_SAMP_0_XY_MIN__MASK 0x00000018 +#define A6XX_TEX_SAMP_0_XY_MIN__SHIFT 3 +static inline uint32_t A6XX_TEX_SAMP_0_XY_MIN(enum a6xx_tex_filter val) +{ + return ((val) << A6XX_TEX_SAMP_0_XY_MIN__SHIFT) & A6XX_TEX_SAMP_0_XY_MIN__MASK; +} +#define A6XX_TEX_SAMP_0_WRAP_S__MASK 0x000000e0 +#define A6XX_TEX_SAMP_0_WRAP_S__SHIFT 5 +static inline uint32_t A6XX_TEX_SAMP_0_WRAP_S(enum a6xx_tex_clamp val) +{ + return ((val) << A6XX_TEX_SAMP_0_WRAP_S__SHIFT) & A6XX_TEX_SAMP_0_WRAP_S__MASK; +} +#define A6XX_TEX_SAMP_0_WRAP_T__MASK 0x00000700 +#define A6XX_TEX_SAMP_0_WRAP_T__SHIFT 8 +static inline uint32_t A6XX_TEX_SAMP_0_WRAP_T(enum a6xx_tex_clamp val) +{ + return ((val) << A6XX_TEX_SAMP_0_WRAP_T__SHIFT) & A6XX_TEX_SAMP_0_WRAP_T__MASK; +} +#define A6XX_TEX_SAMP_0_WRAP_R__MASK 0x00003800 +#define A6XX_TEX_SAMP_0_WRAP_R__SHIFT 11 +static inline uint32_t A6XX_TEX_SAMP_0_WRAP_R(enum a6xx_tex_clamp val) +{ + return ((val) << A6XX_TEX_SAMP_0_WRAP_R__SHIFT) & A6XX_TEX_SAMP_0_WRAP_R__MASK; +} +#define A6XX_TEX_SAMP_0_ANISO__MASK 0x0001c000 +#define A6XX_TEX_SAMP_0_ANISO__SHIFT 14 +static inline uint32_t A6XX_TEX_SAMP_0_ANISO(enum a6xx_tex_aniso val) +{ + return ((val) << A6XX_TEX_SAMP_0_ANISO__SHIFT) & A6XX_TEX_SAMP_0_ANISO__MASK; +} +#define A6XX_TEX_SAMP_0_LOD_BIAS__MASK 0xfff80000 +#define A6XX_TEX_SAMP_0_LOD_BIAS__SHIFT 19 +static inline uint32_t A6XX_TEX_SAMP_0_LOD_BIAS(float val) +{ + return ((((int32_t)(val * 256.0))) << A6XX_TEX_SAMP_0_LOD_BIAS__SHIFT) & A6XX_TEX_SAMP_0_LOD_BIAS__MASK; +} + +#define REG_A6XX_TEX_SAMP_1 0x00000001 +#define A6XX_TEX_SAMP_1_COMPARE_FUNC__MASK 0x0000000e +#define A6XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT 1 +static inline uint32_t A6XX_TEX_SAMP_1_COMPARE_FUNC(enum adreno_compare_func val) +{ + return ((val) << A6XX_TEX_SAMP_1_COMPARE_FUNC__SHIFT) & A6XX_TEX_SAMP_1_COMPARE_FUNC__MASK; +} +#define A6XX_TEX_SAMP_1_CUBEMAPSEAMLESSFILTOFF 0x00000010 +#define A6XX_TEX_SAMP_1_UNNORM_COORDS 0x00000020 +#define A6XX_TEX_SAMP_1_MIPFILTER_LINEAR_FAR 0x00000040 +#define A6XX_TEX_SAMP_1_MAX_LOD__MASK 0x000fff00 +#define A6XX_TEX_SAMP_1_MAX_LOD__SHIFT 8 +static inline uint32_t A6XX_TEX_SAMP_1_MAX_LOD(float val) +{ + return ((((uint32_t)(val * 256.0))) << A6XX_TEX_SAMP_1_MAX_LOD__SHIFT) & A6XX_TEX_SAMP_1_MAX_LOD__MASK; +} +#define A6XX_TEX_SAMP_1_MIN_LOD__MASK 0xfff00000 +#define A6XX_TEX_SAMP_1_MIN_LOD__SHIFT 20 +static inline uint32_t A6XX_TEX_SAMP_1_MIN_LOD(float val) +{ + return ((((uint32_t)(val * 256.0))) << A6XX_TEX_SAMP_1_MIN_LOD__SHIFT) & A6XX_TEX_SAMP_1_MIN_LOD__MASK; +} + +#define REG_A6XX_TEX_SAMP_2 0x00000002 +#define A6XX_TEX_SAMP_2_BCOLOR_OFFSET__MASK 0xfffffff0 +#define A6XX_TEX_SAMP_2_BCOLOR_OFFSET__SHIFT 4 +static inline uint32_t A6XX_TEX_SAMP_2_BCOLOR_OFFSET(uint32_t val) +{ + return ((val) << A6XX_TEX_SAMP_2_BCOLOR_OFFSET__SHIFT) & A6XX_TEX_SAMP_2_BCOLOR_OFFSET__MASK; +} + +#define REG_A6XX_TEX_SAMP_3 0x00000003 + +#define REG_A6XX_TEX_CONST_0 0x00000000 +#define A6XX_TEX_CONST_0_TILE_MODE__MASK 0x00000003 +#define A6XX_TEX_CONST_0_TILE_MODE__SHIFT 0 +static inline uint32_t A6XX_TEX_CONST_0_TILE_MODE(enum a6xx_tile_mode val) +{ + return ((val) << A6XX_TEX_CONST_0_TILE_MODE__SHIFT) & A6XX_TEX_CONST_0_TILE_MODE__MASK; +} +#define A6XX_TEX_CONST_0_SRGB 0x00000004 +#define A6XX_TEX_CONST_0_SWIZ_X__MASK 0x00000070 +#define A6XX_TEX_CONST_0_SWIZ_X__SHIFT 4 +static inline uint32_t A6XX_TEX_CONST_0_SWIZ_X(enum a6xx_tex_swiz val) +{ + return ((val) << A6XX_TEX_CONST_0_SWIZ_X__SHIFT) & A6XX_TEX_CONST_0_SWIZ_X__MASK; +} +#define A6XX_TEX_CONST_0_SWIZ_Y__MASK 0x00000380 +#define A6XX_TEX_CONST_0_SWIZ_Y__SHIFT 7 +static inline uint32_t A6XX_TEX_CONST_0_SWIZ_Y(enum a6xx_tex_swiz val) +{ + return ((val) << A6XX_TEX_CONST_0_SWIZ_Y__SHIFT) & A6XX_TEX_CONST_0_SWIZ_Y__MASK; +} +#define A6XX_TEX_CONST_0_SWIZ_Z__MASK 0x00001c00 +#define A6XX_TEX_CONST_0_SWIZ_Z__SHIFT 10 +static inline uint32_t A6XX_TEX_CONST_0_SWIZ_Z(enum a6xx_tex_swiz val) +{ + return ((val) << A6XX_TEX_CONST_0_SWIZ_Z__SHIFT) & A6XX_TEX_CONST_0_SWIZ_Z__MASK; +} +#define A6XX_TEX_CONST_0_SWIZ_W__MASK 0x0000e000 +#define A6XX_TEX_CONST_0_SWIZ_W__SHIFT 13 +static inline uint32_t A6XX_TEX_CONST_0_SWIZ_W(enum a6xx_tex_swiz val) +{ + return ((val) << A6XX_TEX_CONST_0_SWIZ_W__SHIFT) & A6XX_TEX_CONST_0_SWIZ_W__MASK; +} +#define A6XX_TEX_CONST_0_MIPLVLS__MASK 0x000f0000 +#define A6XX_TEX_CONST_0_MIPLVLS__SHIFT 16 +static inline uint32_t A6XX_TEX_CONST_0_MIPLVLS(uint32_t val) +{ + return ((val) << A6XX_TEX_CONST_0_MIPLVLS__SHIFT) & A6XX_TEX_CONST_0_MIPLVLS__MASK; +} +#define A6XX_TEX_CONST_0_FMT__MASK 0x3fc00000 +#define A6XX_TEX_CONST_0_FMT__SHIFT 22 +static inline uint32_t A6XX_TEX_CONST_0_FMT(enum a6xx_tex_fmt val) +{ + return ((val) << A6XX_TEX_CONST_0_FMT__SHIFT) & A6XX_TEX_CONST_0_FMT__MASK; +} +#define A6XX_TEX_CONST_0_SWAP__MASK 0xc0000000 +#define A6XX_TEX_CONST_0_SWAP__SHIFT 30 +static inline uint32_t A6XX_TEX_CONST_0_SWAP(enum a3xx_color_swap val) +{ + return ((val) << A6XX_TEX_CONST_0_SWAP__SHIFT) & A6XX_TEX_CONST_0_SWAP__MASK; +} + +#define REG_A6XX_TEX_CONST_1 0x00000001 +#define A6XX_TEX_CONST_1_WIDTH__MASK 0x00007fff +#define A6XX_TEX_CONST_1_WIDTH__SHIFT 0 +static inline uint32_t A6XX_TEX_CONST_1_WIDTH(uint32_t val) +{ + return ((val) << A6XX_TEX_CONST_1_WIDTH__SHIFT) & A6XX_TEX_CONST_1_WIDTH__MASK; +} +#define A6XX_TEX_CONST_1_HEIGHT__MASK 0x3fff8000 +#define A6XX_TEX_CONST_1_HEIGHT__SHIFT 15 +static inline uint32_t A6XX_TEX_CONST_1_HEIGHT(uint32_t val) +{ + return ((val) << A6XX_TEX_CONST_1_HEIGHT__SHIFT) & A6XX_TEX_CONST_1_HEIGHT__MASK; +} + +#define REG_A6XX_TEX_CONST_2 0x00000002 +#define A6XX_TEX_CONST_2_FETCHSIZE__MASK 0x0000000f +#define A6XX_TEX_CONST_2_FETCHSIZE__SHIFT 0 +static inline uint32_t A6XX_TEX_CONST_2_FETCHSIZE(enum a6xx_tex_fetchsize val) +{ + return ((val) << A6XX_TEX_CONST_2_FETCHSIZE__SHIFT) & A6XX_TEX_CONST_2_FETCHSIZE__MASK; +} +#define A6XX_TEX_CONST_2_PITCH__MASK 0x1fffff80 +#define A6XX_TEX_CONST_2_PITCH__SHIFT 7 +static inline uint32_t A6XX_TEX_CONST_2_PITCH(uint32_t val) +{ + return ((val) << A6XX_TEX_CONST_2_PITCH__SHIFT) & A6XX_TEX_CONST_2_PITCH__MASK; +} +#define A6XX_TEX_CONST_2_TYPE__MASK 0x60000000 +#define A6XX_TEX_CONST_2_TYPE__SHIFT 29 +static inline uint32_t A6XX_TEX_CONST_2_TYPE(enum a6xx_tex_type val) +{ + return ((val) << A6XX_TEX_CONST_2_TYPE__SHIFT) & A6XX_TEX_CONST_2_TYPE__MASK; +} + +#define REG_A6XX_TEX_CONST_3 0x00000003 +#define A6XX_TEX_CONST_3_ARRAY_PITCH__MASK 0x00003fff +#define A6XX_TEX_CONST_3_ARRAY_PITCH__SHIFT 0 +static inline uint32_t A6XX_TEX_CONST_3_ARRAY_PITCH(uint32_t val) +{ + return ((val >> 12) << A6XX_TEX_CONST_3_ARRAY_PITCH__SHIFT) & A6XX_TEX_CONST_3_ARRAY_PITCH__MASK; +} +#define A6XX_TEX_CONST_3_FLAG 0x10000000 + +#define REG_A6XX_TEX_CONST_4 0x00000004 +#define A6XX_TEX_CONST_4_BASE_LO__MASK 0xffffffe0 +#define A6XX_TEX_CONST_4_BASE_LO__SHIFT 5 +static inline uint32_t A6XX_TEX_CONST_4_BASE_LO(uint32_t val) +{ + return ((val >> 5) << A6XX_TEX_CONST_4_BASE_LO__SHIFT) & A6XX_TEX_CONST_4_BASE_LO__MASK; +} + +#define REG_A6XX_TEX_CONST_5 0x00000005 +#define A6XX_TEX_CONST_5_BASE_HI__MASK 0x0001ffff +#define A6XX_TEX_CONST_5_BASE_HI__SHIFT 0 +static inline uint32_t A6XX_TEX_CONST_5_BASE_HI(uint32_t val) +{ + return ((val) << A6XX_TEX_CONST_5_BASE_HI__SHIFT) & A6XX_TEX_CONST_5_BASE_HI__MASK; +} +#define A6XX_TEX_CONST_5_DEPTH__MASK 0x3ffe0000 +#define A6XX_TEX_CONST_5_DEPTH__SHIFT 17 +static inline uint32_t A6XX_TEX_CONST_5_DEPTH(uint32_t val) +{ + return ((val) << A6XX_TEX_CONST_5_DEPTH__SHIFT) & A6XX_TEX_CONST_5_DEPTH__MASK; +} + +#define REG_A6XX_TEX_CONST_6 0x00000006 + +#define REG_A6XX_TEX_CONST_7 0x00000007 +#define A6XX_TEX_CONST_7_FLAG_LO__MASK 0xffffffe0 +#define A6XX_TEX_CONST_7_FLAG_LO__SHIFT 5 +static inline uint32_t A6XX_TEX_CONST_7_FLAG_LO(uint32_t val) +{ + return ((val >> 5) << A6XX_TEX_CONST_7_FLAG_LO__SHIFT) & A6XX_TEX_CONST_7_FLAG_LO__MASK; +} + +#define REG_A6XX_TEX_CONST_8 0x00000008 +#define A6XX_TEX_CONST_8_BASE_HI__MASK 0x0001ffff +#define A6XX_TEX_CONST_8_BASE_HI__SHIFT 0 +static inline uint32_t A6XX_TEX_CONST_8_BASE_HI(uint32_t val) +{ + return ((val) << A6XX_TEX_CONST_8_BASE_HI__SHIFT) & A6XX_TEX_CONST_8_BASE_HI__MASK; +} + +#define REG_A6XX_TEX_CONST_9 0x00000009 + +#define REG_A6XX_TEX_CONST_10 0x0000000a + +#define REG_A6XX_TEX_CONST_11 0x0000000b + +#define REG_A6XX_TEX_CONST_12 0x0000000c + +#define REG_A6XX_TEX_CONST_13 0x0000000d + +#define REG_A6XX_TEX_CONST_14 0x0000000e + +#define REG_A6XX_TEX_CONST_15 0x0000000f + + +#endif /* A6XX_XML */ diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.c b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c new file mode 100644 index 000000000000..bbb8126ec5c5 --- /dev/null +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.c @@ -0,0 +1,1207 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */ + +#include <linux/clk.h> +#include <linux/iopoll.h> +#include <linux/pm_opp.h> +#include <soc/qcom/cmd-db.h> + +#include "a6xx_gpu.h" +#include "a6xx_gmu.xml.h" + +static irqreturn_t a6xx_gmu_irq(int irq, void *data) +{ + struct a6xx_gmu *gmu = data; + u32 status; + + status = gmu_read(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS); + gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, status); + + if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE) { + dev_err_ratelimited(gmu->dev, "GMU watchdog expired\n"); + + /* Temporary until we can recover safely */ + BUG(); + } + + if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR) + dev_err_ratelimited(gmu->dev, "GMU AHB bus error\n"); + + if (status & A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR) + dev_err_ratelimited(gmu->dev, "GMU fence error: 0x%x\n", + gmu_read(gmu, REG_A6XX_GMU_AHB_FENCE_STATUS)); + + return IRQ_HANDLED; +} + +static irqreturn_t a6xx_hfi_irq(int irq, void *data) +{ + struct a6xx_gmu *gmu = data; + u32 status; + + status = gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO); + gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, status); + + if (status & A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ) + tasklet_schedule(&gmu->hfi_tasklet); + + if (status & A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) { + dev_err_ratelimited(gmu->dev, "GMU firmware fault\n"); + + /* Temporary until we can recover safely */ + BUG(); + } + + return IRQ_HANDLED; +} + +/* Check to see if the GX rail is still powered */ +static bool a6xx_gmu_gx_is_on(struct a6xx_gmu *gmu) +{ + u32 val = gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS); + + return !(val & + (A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF | + A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF)); +} + +static int a6xx_gmu_set_freq(struct a6xx_gmu *gmu, int index) +{ + gmu_write(gmu, REG_A6XX_GMU_DCVS_ACK_OPTION, 0); + + gmu_write(gmu, REG_A6XX_GMU_DCVS_PERF_SETTING, + ((index << 24) & 0xff) | (3 & 0xf)); + + /* + * Send an invalid index as a vote for the bus bandwidth and let the + * firmware decide on the right vote + */ + gmu_write(gmu, REG_A6XX_GMU_DCVS_BW_SETTING, 0xff); + + /* Set and clear the OOB for DCVS to trigger the GMU */ + a6xx_gmu_set_oob(gmu, GMU_OOB_DCVS_SET); + a6xx_gmu_clear_oob(gmu, GMU_OOB_DCVS_SET); + + return gmu_read(gmu, REG_A6XX_GMU_DCVS_RETURN); +} + +static bool a6xx_gmu_check_idle_level(struct a6xx_gmu *gmu) +{ + u32 val; + int local = gmu->idle_level; + + /* SPTP and IFPC both report as IFPC */ + if (gmu->idle_level == GMU_IDLE_STATE_SPTP) + local = GMU_IDLE_STATE_IFPC; + + val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE); + + if (val == local) { + if (gmu->idle_level != GMU_IDLE_STATE_IFPC || + !a6xx_gmu_gx_is_on(gmu)) + return true; + } + + return false; +} + +/* Wait for the GMU to get to its most idle state */ +int a6xx_gmu_wait_for_idle(struct a6xx_gpu *a6xx_gpu) +{ + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + + return spin_until(a6xx_gmu_check_idle_level(gmu)); +} + +static int a6xx_gmu_start(struct a6xx_gmu *gmu) +{ + int ret; + u32 val; + + gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 1); + gmu_write(gmu, REG_A6XX_GMU_CM3_SYSRESET, 0); + + ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, val, + val == 0xbabeface, 100, 10000); + + if (ret) + dev_err(gmu->dev, "GMU firmware initialization timed out\n"); + + return ret; +} + +static int a6xx_gmu_hfi_start(struct a6xx_gmu *gmu) +{ + u32 val; + int ret; + + gmu_rmw(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, + A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ, 0); + + gmu_write(gmu, REG_A6XX_GMU_HFI_CTRL_INIT, 1); + + ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_HFI_CTRL_STATUS, val, + val & 1, 100, 10000); + if (ret) + dev_err(gmu->dev, "Unable to start the HFI queues\n"); + + return ret; +} + +/* Trigger a OOB (out of band) request to the GMU */ +int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) +{ + int ret; + u32 val; + int request, ack; + const char *name; + + switch (state) { + case GMU_OOB_GPU_SET: + request = GMU_OOB_GPU_SET_REQUEST; + ack = GMU_OOB_GPU_SET_ACK; + name = "GPU_SET"; + break; + case GMU_OOB_BOOT_SLUMBER: + request = GMU_OOB_BOOT_SLUMBER_REQUEST; + ack = GMU_OOB_BOOT_SLUMBER_ACK; + name = "BOOT_SLUMBER"; + break; + case GMU_OOB_DCVS_SET: + request = GMU_OOB_DCVS_REQUEST; + ack = GMU_OOB_DCVS_ACK; + name = "GPU_DCVS"; + break; + default: + return -EINVAL; + } + + /* Trigger the equested OOB operation */ + gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 1 << request); + + /* Wait for the acknowledge interrupt */ + ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO, val, + val & (1 << ack), 100, 10000); + + if (ret) + dev_err(gmu->dev, + "Timeout waiting for GMU OOB set %s: 0x%x\n", + name, + gmu_read(gmu, REG_A6XX_GMU_GMU2HOST_INTR_INFO)); + + /* Clear the acknowledge interrupt */ + gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, 1 << ack); + + return ret; +} + +/* Clear a pending OOB state in the GMU */ +void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state) +{ + switch (state) { + case GMU_OOB_GPU_SET: + gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, + 1 << GMU_OOB_GPU_SET_CLEAR); + break; + case GMU_OOB_BOOT_SLUMBER: + gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, + 1 << GMU_OOB_BOOT_SLUMBER_CLEAR); + break; + case GMU_OOB_DCVS_SET: + gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, + 1 << GMU_OOB_DCVS_CLEAR); + break; + } +} + +/* Enable CPU control of SPTP power power collapse */ +static int a6xx_sptprac_enable(struct a6xx_gmu *gmu) +{ + int ret; + u32 val; + + gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778000); + + ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val, + (val & 0x38) == 0x28, 1, 100); + + if (ret) { + dev_err(gmu->dev, "Unable to power on SPTPRAC: 0x%x\n", + gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS)); + } + + return 0; +} + +/* Disable CPU control of SPTP power power collapse */ +static void a6xx_sptprac_disable(struct a6xx_gmu *gmu) +{ + u32 val; + int ret; + + /* Make sure retention is on */ + gmu_rmw(gmu, REG_A6XX_GPU_CC_GX_GDSCR, 0, (1 << 11)); + + gmu_write(gmu, REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL, 0x778001); + + ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS, val, + (val & 0x04), 100, 10000); + + if (ret) + dev_err(gmu->dev, "failed to power off SPTPRAC: 0x%x\n", + gmu_read(gmu, REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS)); +} + +/* Let the GMU know we are starting a boot sequence */ +static int a6xx_gmu_gfx_rail_on(struct a6xx_gmu *gmu) +{ + u32 vote; + + /* Let the GMU know we are getting ready for boot */ + gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 0); + + /* Choose the "default" power level as the highest available */ + vote = gmu->gx_arc_votes[gmu->nr_gpu_freqs - 1]; + + gmu_write(gmu, REG_A6XX_GMU_GX_VOTE_IDX, vote & 0xff); + gmu_write(gmu, REG_A6XX_GMU_MX_VOTE_IDX, (vote >> 8) & 0xff); + + /* Let the GMU know the boot sequence has started */ + return a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER); +} + +/* Let the GMU know that we are about to go into slumber */ +static int a6xx_gmu_notify_slumber(struct a6xx_gmu *gmu) +{ + int ret; + + /* Disable the power counter so the GMU isn't busy */ + gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 0); + + /* Disable SPTP_PC if the CPU is responsible for it */ + if (gmu->idle_level < GMU_IDLE_STATE_SPTP) + a6xx_sptprac_disable(gmu); + + /* Tell the GMU to get ready to slumber */ + gmu_write(gmu, REG_A6XX_GMU_BOOT_SLUMBER_OPTION, 1); + + ret = a6xx_gmu_set_oob(gmu, GMU_OOB_BOOT_SLUMBER); + a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER); + + if (!ret) { + /* Check to see if the GMU really did slumber */ + if (gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE) + != 0x0f) { + dev_err(gmu->dev, "The GMU did not go into slumber\n"); + ret = -ETIMEDOUT; + } + } + + /* Put fence into allow mode */ + gmu_write(gmu, REG_A6XX_GMU_AO_AHB_FENCE_CTRL, 0); + return ret; +} + +static int a6xx_rpmh_start(struct a6xx_gmu *gmu) +{ + int ret; + u32 val; + + gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1 << 1); + /* Wait for the register to finish posting */ + wmb(); + + ret = gmu_poll_timeout(gmu, REG_A6XX_GMU_RSCC_CONTROL_ACK, val, + val & (1 << 1), 100, 10000); + if (ret) { + dev_err(gmu->dev, "Unable to power on the GPU RSC\n"); + return ret; + } + + ret = gmu_poll_timeout(gmu, REG_A6XX_RSCC_SEQ_BUSY_DRV0, val, + !val, 100, 10000); + + if (!ret) { + gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0); + + /* Re-enable the power counter */ + gmu_write(gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, 1); + return 0; + } + + dev_err(gmu->dev, "GPU RSC sequence stuck while waking up the GPU\n"); + return ret; +} + +static void a6xx_rpmh_stop(struct a6xx_gmu *gmu) +{ + int ret; + u32 val; + + gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 1); + + ret = gmu_poll_timeout(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, + val, val & (1 << 16), 100, 10000); + if (ret) + dev_err(gmu->dev, "Unable to power off the GPU RSC\n"); + + gmu_write(gmu, REG_A6XX_GMU_RSCC_CONTROL_REQ, 0); +} + +static void a6xx_gmu_rpmh_init(struct a6xx_gmu *gmu) +{ + /* Disable SDE clock gating */ + gmu_write(gmu, REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0, BIT(24)); + + /* Setup RSC PDC handshake for sleep and wakeup */ + gmu_write(gmu, REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0, 1); + gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA, 0); + gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR, 0); + gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 2, 0); + gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 2, 0); + gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA + 4, 0x80000000); + gmu_write(gmu, REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR + 4, 0); + gmu_write(gmu, REG_A6XX_RSCC_OVERRIDE_START_ADDR, 0); + gmu_write(gmu, REG_A6XX_RSCC_PDC_SEQ_START_ADDR, 0x4520); + gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_LO, 0x4510); + gmu_write(gmu, REG_A6XX_RSCC_PDC_MATCH_VALUE_HI, 0x4514); + + /* Load RSC sequencer uCode for sleep and wakeup */ + gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0, 0xa7a506a0); + gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 1, 0xa1e6a6e7); + gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 2, 0xa2e081e1); + gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 3, 0xe9a982e2); + gmu_write(gmu, REG_A6XX_RSCC_SEQ_MEM_0_DRV0 + 4, 0x0020e8a8); + + /* Load PDC sequencer uCode for power up and power down sequence */ + pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0, 0xfebea1e1); + pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 1, 0xa5a4a3a2); + pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 2, 0x8382a6e0); + pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 3, 0xbce3e284); + pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_MEM_0 + 4, 0x002081fc); + + /* Set TCS commands used by PDC sequence for low power modes */ + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD_ENABLE_BANK, 7); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD_WAIT_FOR_CMPL_BANK, 0); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CONTROL, 0); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID, 0x10108); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR, 0x30010); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA, 1); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 4, 0x10108); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 4, 0x30000); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 4, 0x0); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_MSGID + 8, 0x10108); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_ADDR + 8, 0x30080); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS1_CMD0_DATA + 8, 0x0); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD_ENABLE_BANK, 7); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD_WAIT_FOR_CMPL_BANK, 0); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CONTROL, 0); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID, 0x10108); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR, 0x30010); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA, 2); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 4, 0x10108); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 4, 0x30000); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 4, 0x3); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_MSGID + 8, 0x10108); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_ADDR + 8, 0x30080); + pdc_write(gmu, REG_A6XX_PDC_GPU_TCS3_CMD0_DATA + 8, 0x3); + + /* Setup GPU PDC */ + pdc_write(gmu, REG_A6XX_PDC_GPU_SEQ_START_ADDR, 0); + pdc_write(gmu, REG_A6XX_PDC_GPU_ENABLE_PDC, 0x80000001); + + /* ensure no writes happen before the uCode is fully written */ + wmb(); +} + +/* + * The lowest 16 bits of this value are the number of XO clock cycles for main + * hysteresis which is set at 0x1680 cycles (300 us). The higher 16 bits are + * for the shorter hysteresis that happens after main - this is 0xa (.5 us) + */ + +#define GMU_PWR_COL_HYST 0x000a1680 + +/* Set up the idle state for the GMU */ +static void a6xx_gmu_power_config(struct a6xx_gmu *gmu) +{ + /* Disable GMU WB/RB buffer */ + gmu_write(gmu, REG_A6XX_GMU_SYS_BUS_CONFIG, 0x1); + + gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0x9c40400); + + switch (gmu->idle_level) { + case GMU_IDLE_STATE_IFPC: + gmu_write(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST, + GMU_PWR_COL_HYST); + gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0, + A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE | + A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE); + /* Fall through */ + case GMU_IDLE_STATE_SPTP: + gmu_write(gmu, REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST, + GMU_PWR_COL_HYST); + gmu_rmw(gmu, REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL, 0, + A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE | + A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE); + } + + /* Enable RPMh GPU client */ + gmu_rmw(gmu, REG_A6XX_GMU_RPMH_CTRL, 0, + A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE | + A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE | + A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE | + A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE | + A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE | + A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE); +} + +static int a6xx_gmu_fw_start(struct a6xx_gmu *gmu, unsigned int state) +{ + static bool rpmh_init; + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + int i, ret; + u32 chipid; + u32 *image; + + if (state == GMU_WARM_BOOT) { + ret = a6xx_rpmh_start(gmu); + if (ret) + return ret; + } else { + if (WARN(!adreno_gpu->fw[ADRENO_FW_GMU], + "GMU firmware is not loaded\n")) + return -ENOENT; + + /* Sanity check the size of the firmware that was loaded */ + if (adreno_gpu->fw[ADRENO_FW_GMU]->size > 0x8000) { + dev_err(gmu->dev, + "GMU firmware is bigger than the available region\n"); + return -EINVAL; + } + + /* Turn on register retention */ + gmu_write(gmu, REG_A6XX_GMU_GENERAL_7, 1); + + /* We only need to load the RPMh microcode once */ + if (!rpmh_init) { + a6xx_gmu_rpmh_init(gmu); + rpmh_init = true; + } else if (state != GMU_RESET) { + ret = a6xx_rpmh_start(gmu); + if (ret) + return ret; + } + + image = (u32 *) adreno_gpu->fw[ADRENO_FW_GMU]->data; + + for (i = 0; i < adreno_gpu->fw[ADRENO_FW_GMU]->size >> 2; i++) + gmu_write(gmu, REG_A6XX_GMU_CM3_ITCM_START + i, + image[i]); + } + + gmu_write(gmu, REG_A6XX_GMU_CM3_FW_INIT_RESULT, 0); + gmu_write(gmu, REG_A6XX_GMU_CM3_BOOT_CONFIG, 0x02); + + /* Write the iova of the HFI table */ + gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_ADDR, gmu->hfi->iova); + gmu_write(gmu, REG_A6XX_GMU_HFI_QTBL_INFO, 1); + + gmu_write(gmu, REG_A6XX_GMU_AHB_FENCE_RANGE_0, + (1 << 31) | (0xa << 18) | (0xa0)); + + chipid = adreno_gpu->rev.core << 24; + chipid |= adreno_gpu->rev.major << 16; + chipid |= adreno_gpu->rev.minor << 12; + chipid |= adreno_gpu->rev.patchid << 8; + + gmu_write(gmu, REG_A6XX_GMU_HFI_SFR_ADDR, chipid); + + /* Set up the lowest idle level on the GMU */ + a6xx_gmu_power_config(gmu); + + ret = a6xx_gmu_start(gmu); + if (ret) + return ret; + + ret = a6xx_gmu_gfx_rail_on(gmu); + if (ret) + return ret; + + /* Enable SPTP_PC if the CPU is responsible for it */ + if (gmu->idle_level < GMU_IDLE_STATE_SPTP) { + ret = a6xx_sptprac_enable(gmu); + if (ret) + return ret; + } + + ret = a6xx_gmu_hfi_start(gmu); + if (ret) + return ret; + + /* FIXME: Do we need this wmb() here? */ + wmb(); + + return 0; +} + +#define A6XX_HFI_IRQ_MASK \ + (A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ | \ + A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT) + +#define A6XX_GMU_IRQ_MASK \ + (A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE | \ + A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR | \ + A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR) + +static void a6xx_gmu_irq_enable(struct a6xx_gmu *gmu) +{ + gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR, ~0); + gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_CLR, ~0); + + gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, + ~A6XX_GMU_IRQ_MASK); + gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, + ~A6XX_HFI_IRQ_MASK); + + enable_irq(gmu->gmu_irq); + enable_irq(gmu->hfi_irq); +} + +static void a6xx_gmu_irq_disable(struct a6xx_gmu *gmu) +{ + disable_irq(gmu->gmu_irq); + disable_irq(gmu->hfi_irq); + + gmu_write(gmu, REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK, ~0); + gmu_write(gmu, REG_A6XX_GMU_GMU2HOST_INTR_MASK, ~0); +} + +int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu) +{ + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + int ret; + u32 val; + + /* Flush all the queues */ + a6xx_hfi_stop(gmu); + + /* Stop the interrupts */ + a6xx_gmu_irq_disable(gmu); + + /* Force off SPTP in case the GMU is managing it */ + a6xx_sptprac_disable(gmu); + + /* Make sure there are no outstanding RPMh votes */ + gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS0_DRV0_STATUS, val, + (val & 1), 100, 10000); + gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS1_DRV0_STATUS, val, + (val & 1), 100, 10000); + gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS2_DRV0_STATUS, val, + (val & 1), 100, 10000); + gmu_poll_timeout(gmu, REG_A6XX_RSCC_TCS3_DRV0_STATUS, val, + (val & 1), 100, 1000); + + /* Force off the GX GSDC */ + regulator_force_disable(gmu->gx); + + /* Disable the resources */ + clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks); + pm_runtime_put_sync(gmu->dev); + + /* Re-enable the resources */ + pm_runtime_get_sync(gmu->dev); + + /* Use a known rate to bring up the GMU */ + clk_set_rate(gmu->core_clk, 200000000); + ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks); + if (ret) + goto out; + + a6xx_gmu_irq_enable(gmu); + + ret = a6xx_gmu_fw_start(gmu, GMU_RESET); + if (!ret) + ret = a6xx_hfi_start(gmu, GMU_COLD_BOOT); + + /* Set the GPU back to the highest power frequency */ + a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1); + +out: + if (ret) + a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER); + + return ret; +} + +int a6xx_gmu_resume(struct a6xx_gpu *a6xx_gpu) +{ + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + int status, ret; + + if (WARN(!gmu->mmio, "The GMU is not set up yet\n")) + return 0; + + /* Turn on the resources */ + pm_runtime_get_sync(gmu->dev); + + /* Use a known rate to bring up the GMU */ + clk_set_rate(gmu->core_clk, 200000000); + ret = clk_bulk_prepare_enable(gmu->nr_clocks, gmu->clocks); + if (ret) + goto out; + + a6xx_gmu_irq_enable(gmu); + + /* Check to see if we are doing a cold or warm boot */ + status = gmu_read(gmu, REG_A6XX_GMU_GENERAL_7) == 1 ? + GMU_WARM_BOOT : GMU_COLD_BOOT; + + ret = a6xx_gmu_fw_start(gmu, status); + if (ret) + goto out; + + ret = a6xx_hfi_start(gmu, status); + + /* Set the GPU to the highest power frequency */ + a6xx_gmu_set_freq(gmu, gmu->nr_gpu_freqs - 1); + +out: + /* Make sure to turn off the boot OOB request on error */ + if (ret) + a6xx_gmu_clear_oob(gmu, GMU_OOB_BOOT_SLUMBER); + + return ret; +} + +bool a6xx_gmu_isidle(struct a6xx_gmu *gmu) +{ + u32 reg; + + if (!gmu->mmio) + return true; + + reg = gmu_read(gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS); + + if (reg & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB) + return false; + + return true; +} + +int a6xx_gmu_stop(struct a6xx_gpu *a6xx_gpu) +{ + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + u32 val; + + /* + * The GMU may still be in slumber unless the GPU started so check and + * skip putting it back into slumber if so + */ + val = gmu_read(gmu, REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE); + + if (val != 0xf) { + int ret = a6xx_gmu_wait_for_idle(a6xx_gpu); + + /* Temporary until we can recover safely */ + BUG_ON(ret); + + /* tell the GMU we want to slumber */ + a6xx_gmu_notify_slumber(gmu); + + ret = gmu_poll_timeout(gmu, + REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS, val, + !(val & A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB), + 100, 10000); + + /* + * Let the user know we failed to slumber but don't worry too + * much because we are powering down anyway + */ + + if (ret) + dev_err(gmu->dev, + "Unable to slumber GMU: status = 0%x/0%x\n", + gmu_read(gmu, + REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS), + gmu_read(gmu, + REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2)); + } + + /* Turn off HFI */ + a6xx_hfi_stop(gmu); + + /* Stop the interrupts and mask the hardware */ + a6xx_gmu_irq_disable(gmu); + + /* Tell RPMh to power off the GPU */ + a6xx_rpmh_stop(gmu); + + clk_bulk_disable_unprepare(gmu->nr_clocks, gmu->clocks); + + pm_runtime_put_sync(gmu->dev); + + return 0; +} + +static void a6xx_gmu_memory_free(struct a6xx_gmu *gmu, struct a6xx_gmu_bo *bo) +{ + int count, i; + u64 iova; + + if (IS_ERR_OR_NULL(bo)) + return; + + count = bo->size >> PAGE_SHIFT; + iova = bo->iova; + + for (i = 0; i < count; i++, iova += PAGE_SIZE) { + iommu_unmap(gmu->domain, iova, PAGE_SIZE); + __free_pages(bo->pages[i], 0); + } + + kfree(bo->pages); + kfree(bo); +} + +static struct a6xx_gmu_bo *a6xx_gmu_memory_alloc(struct a6xx_gmu *gmu, + size_t size) +{ + struct a6xx_gmu_bo *bo; + int ret, count, i; + + bo = kzalloc(sizeof(*bo), GFP_KERNEL); + if (!bo) + return ERR_PTR(-ENOMEM); + + bo->size = PAGE_ALIGN(size); + + count = bo->size >> PAGE_SHIFT; + + bo->pages = kcalloc(count, sizeof(struct page *), GFP_KERNEL); + if (!bo->pages) { + kfree(bo); + return ERR_PTR(-ENOMEM); + } + + for (i = 0; i < count; i++) { + bo->pages[i] = alloc_page(GFP_KERNEL); + if (!bo->pages[i]) + goto err; + } + + bo->iova = gmu->uncached_iova_base; + + for (i = 0; i < count; i++) { + ret = iommu_map(gmu->domain, + bo->iova + (PAGE_SIZE * i), + page_to_phys(bo->pages[i]), PAGE_SIZE, + IOMMU_READ | IOMMU_WRITE); + + if (ret) { + dev_err(gmu->dev, "Unable to map GMU buffer object\n"); + + for (i = i - 1 ; i >= 0; i--) + iommu_unmap(gmu->domain, + bo->iova + (PAGE_SIZE * i), + PAGE_SIZE); + + goto err; + } + } + + bo->virt = vmap(bo->pages, count, VM_IOREMAP, + pgprot_writecombine(PAGE_KERNEL)); + if (!bo->virt) + goto err; + + /* Align future IOVA addresses on 1MB boundaries */ + gmu->uncached_iova_base += ALIGN(size, SZ_1M); + + return bo; + +err: + for (i = 0; i < count; i++) { + if (bo->pages[i]) + __free_pages(bo->pages[i], 0); + } + + kfree(bo->pages); + kfree(bo); + + return ERR_PTR(-ENOMEM); +} + +static int a6xx_gmu_memory_probe(struct a6xx_gmu *gmu) +{ + int ret; + + /* + * The GMU address space is hardcoded to treat the range + * 0x60000000 - 0x80000000 as un-cached memory. All buffers shared + * between the GMU and the CPU will live in this space + */ + gmu->uncached_iova_base = 0x60000000; + + + gmu->domain = iommu_domain_alloc(&platform_bus_type); + if (!gmu->domain) + return -ENODEV; + + ret = iommu_attach_device(gmu->domain, gmu->dev); + + if (ret) { + iommu_domain_free(gmu->domain); + gmu->domain = NULL; + } + + return ret; +} + +/* Get the list of RPMh voltage levels from cmd-db */ +static int a6xx_gmu_rpmh_arc_cmds(const char *id, void *vals, int size) +{ + u32 len = cmd_db_read_aux_data_len(id); + + if (!len) + return 0; + + if (WARN_ON(len > size)) + return -EINVAL; + + cmd_db_read_aux_data(id, vals, len); + + /* + * The data comes back as an array of unsigned shorts so adjust the + * count accordingly + */ + return len >> 1; +} + +/* Return the 'arc-level' for the given frequency */ +static u32 a6xx_gmu_get_arc_level(struct device *dev, unsigned long freq) +{ + struct dev_pm_opp *opp; + struct device_node *np; + u32 val = 0; + + if (!freq) + return 0; + + opp = dev_pm_opp_find_freq_exact(dev, freq, true); + if (IS_ERR(opp)) + return 0; + + np = dev_pm_opp_get_of_node(opp); + + if (np) { + of_property_read_u32(np, "qcom,level", &val); + of_node_put(np); + } + + dev_pm_opp_put(opp); + + return val; +} + +static int a6xx_gmu_rpmh_arc_votes_init(struct device *dev, u32 *votes, + unsigned long *freqs, int freqs_count, + u16 *pri, int pri_count, + u16 *sec, int sec_count) +{ + int i, j; + + /* Construct a vote for each frequency */ + for (i = 0; i < freqs_count; i++) { + u8 pindex = 0, sindex = 0; + u32 level = a6xx_gmu_get_arc_level(dev, freqs[i]); + + /* Get the primary index that matches the arc level */ + for (j = 0; j < pri_count; j++) { + if (pri[j] >= level) { + pindex = j; + break; + } + } + + if (j == pri_count) { + dev_err(dev, + "Level %u not found in in the RPMh list\n", + level); + dev_err(dev, "Available levels:\n"); + for (j = 0; j < pri_count; j++) + dev_err(dev, " %u\n", pri[j]); + + return -EINVAL; + } + + /* + * Look for a level in in the secondary list that matches. If + * nothing fits, use the maximum non zero vote + */ + + for (j = 0; j < sec_count; j++) { + if (sec[j] >= level) { + sindex = j; + break; + } else if (sec[j]) { + sindex = j; + } + } + + /* Construct the vote */ + votes[i] = ((pri[pindex] & 0xffff) << 16) | + (sindex << 8) | pindex; + } + + return 0; +} + +/* + * The GMU votes with the RPMh for itself and on behalf of the GPU but we need + * to construct the list of votes on the CPU and send it over. Query the RPMh + * voltage levels and build the votes + */ + +static int a6xx_gmu_rpmh_votes_init(struct a6xx_gmu *gmu) +{ + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; + + u16 gx[16], cx[16], mx[16]; + u32 gxcount, cxcount, mxcount; + int ret; + + /* Get the list of available voltage levels for each component */ + gxcount = a6xx_gmu_rpmh_arc_cmds("gfx.lvl", gx, sizeof(gx)); + cxcount = a6xx_gmu_rpmh_arc_cmds("cx.lvl", cx, sizeof(cx)); + mxcount = a6xx_gmu_rpmh_arc_cmds("mx.lvl", mx, sizeof(mx)); + + /* Build the GX votes */ + ret = a6xx_gmu_rpmh_arc_votes_init(&gpu->pdev->dev, gmu->gx_arc_votes, + gmu->gpu_freqs, gmu->nr_gpu_freqs, + gx, gxcount, mx, mxcount); + + /* Build the CX votes */ + ret |= a6xx_gmu_rpmh_arc_votes_init(gmu->dev, gmu->cx_arc_votes, + gmu->gmu_freqs, gmu->nr_gmu_freqs, + cx, cxcount, mx, mxcount); + + return ret; +} + +static int a6xx_gmu_build_freq_table(struct device *dev, unsigned long *freqs, + u32 size) +{ + int count = dev_pm_opp_get_opp_count(dev); + struct dev_pm_opp *opp; + int i, index = 0; + unsigned long freq = 1; + + /* + * The OPP table doesn't contain the "off" frequency level so we need to + * add 1 to the table size to account for it + */ + + if (WARN(count + 1 > size, + "The GMU frequency table is being truncated\n")) + count = size - 1; + + /* Set the "off" frequency */ + freqs[index++] = 0; + + for (i = 0; i < count; i++) { + opp = dev_pm_opp_find_freq_ceil(dev, &freq); + if (IS_ERR(opp)) + break; + + dev_pm_opp_put(opp); + freqs[index++] = freq++; + } + + return index; +} + +static int a6xx_gmu_pwrlevels_probe(struct a6xx_gmu *gmu) +{ + struct a6xx_gpu *a6xx_gpu = container_of(gmu, struct a6xx_gpu, gmu); + struct adreno_gpu *adreno_gpu = &a6xx_gpu->base; + struct msm_gpu *gpu = &adreno_gpu->base; + + int ret = 0; + + /* + * The GMU handles its own frequency switching so build a list of + * available frequencies to send during initialization + */ + ret = dev_pm_opp_of_add_table(gmu->dev); + if (ret) { + dev_err(gmu->dev, "Unable to set the OPP table for the GMU\n"); + return ret; + } + + gmu->nr_gmu_freqs = a6xx_gmu_build_freq_table(gmu->dev, + gmu->gmu_freqs, ARRAY_SIZE(gmu->gmu_freqs)); + + /* + * The GMU also handles GPU frequency switching so build a list + * from the GPU OPP table + */ + gmu->nr_gpu_freqs = a6xx_gmu_build_freq_table(&gpu->pdev->dev, + gmu->gpu_freqs, ARRAY_SIZE(gmu->gpu_freqs)); + + /* Build the list of RPMh votes that we'll send to the GMU */ + return a6xx_gmu_rpmh_votes_init(gmu); +} + +static int a6xx_gmu_clocks_probe(struct a6xx_gmu *gmu) +{ + int ret = msm_clk_bulk_get(gmu->dev, &gmu->clocks); + + if (ret < 1) + return ret; + + gmu->nr_clocks = ret; + + gmu->core_clk = msm_clk_bulk_get_clock(gmu->clocks, + gmu->nr_clocks, "gmu"); + + return 0; +} + +static void __iomem *a6xx_gmu_get_mmio(struct platform_device *pdev, + const char *name) +{ + void __iomem *ret; + struct resource *res = platform_get_resource_byname(pdev, + IORESOURCE_MEM, name); + + if (!res) { + dev_err(&pdev->dev, "Unable to find the %s registers\n", name); + return ERR_PTR(-EINVAL); + } + + ret = devm_ioremap(&pdev->dev, res->start, resource_size(res)); + if (!ret) { + dev_err(&pdev->dev, "Unable to map the %s registers\n", name); + return ERR_PTR(-EINVAL); + } + + return ret; +} + +static int a6xx_gmu_get_irq(struct a6xx_gmu *gmu, struct platform_device *pdev, + const char *name, irq_handler_t handler) +{ + int irq, ret; + + irq = platform_get_irq_byname(pdev, name); + + ret = devm_request_irq(&pdev->dev, irq, handler, IRQF_TRIGGER_HIGH, + name, gmu); + if (ret) { + dev_err(&pdev->dev, "Unable to get interrupt %s\n", name); + return ret; + } + + disable_irq(irq); + + return irq; +} + +void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu) +{ + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + + if (IS_ERR_OR_NULL(gmu->mmio)) + return; + + pm_runtime_disable(gmu->dev); + a6xx_gmu_stop(a6xx_gpu); + + a6xx_gmu_irq_disable(gmu); + a6xx_gmu_memory_free(gmu, gmu->hfi); + + iommu_detach_device(gmu->domain, gmu->dev); + + iommu_domain_free(gmu->domain); +} + +int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node) +{ + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + struct platform_device *pdev = of_find_device_by_node(node); + int ret; + + if (!pdev) + return -ENODEV; + + gmu->dev = &pdev->dev; + + of_dma_configure(gmu->dev, node, false); + + /* Fow now, don't do anything fancy until we get our feet under us */ + gmu->idle_level = GMU_IDLE_STATE_ACTIVE; + + pm_runtime_enable(gmu->dev); + gmu->gx = devm_regulator_get(gmu->dev, "vdd"); + + /* Get the list of clocks */ + ret = a6xx_gmu_clocks_probe(gmu); + if (ret) + return ret; + + /* Set up the IOMMU context bank */ + ret = a6xx_gmu_memory_probe(gmu); + if (ret) + return ret; + + /* Allocate memory for for the HFI queues */ + gmu->hfi = a6xx_gmu_memory_alloc(gmu, SZ_16K); + if (IS_ERR(gmu->hfi)) + goto err; + + /* Allocate memory for the GMU debug region */ + gmu->debug = a6xx_gmu_memory_alloc(gmu, SZ_16K); + if (IS_ERR(gmu->debug)) + goto err; + + /* Map the GMU registers */ + gmu->mmio = a6xx_gmu_get_mmio(pdev, "gmu"); + + /* Map the GPU power domain controller registers */ + gmu->pdc_mmio = a6xx_gmu_get_mmio(pdev, "gmu_pdc"); + + if (IS_ERR(gmu->mmio) || IS_ERR(gmu->pdc_mmio)) + goto err; + + /* Get the HFI and GMU interrupts */ + gmu->hfi_irq = a6xx_gmu_get_irq(gmu, pdev, "hfi", a6xx_hfi_irq); + gmu->gmu_irq = a6xx_gmu_get_irq(gmu, pdev, "gmu", a6xx_gmu_irq); + + if (gmu->hfi_irq < 0 || gmu->gmu_irq < 0) + goto err; + + /* Set up a tasklet to handle GMU HFI responses */ + tasklet_init(&gmu->hfi_tasklet, a6xx_hfi_task, (unsigned long) gmu); + + /* Get the power levels for the GMU and GPU */ + a6xx_gmu_pwrlevels_probe(gmu); + + /* Set up the HFI queues */ + a6xx_hfi_init(gmu); + + return 0; +err: + a6xx_gmu_memory_free(gmu, gmu->hfi); + + if (gmu->domain) { + iommu_detach_device(gmu->domain, gmu->dev); + + iommu_domain_free(gmu->domain); + } + + return -ENODEV; +} diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h new file mode 100644 index 000000000000..d9a386c18799 --- /dev/null +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.h @@ -0,0 +1,162 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2017 The Linux Foundation. All rights reserved. */ + +#ifndef _A6XX_GMU_H_ +#define _A6XX_GMU_H_ + +#include <linux/interrupt.h> +#include "msm_drv.h" +#include "a6xx_hfi.h" + +struct a6xx_gmu_bo { + void *virt; + size_t size; + u64 iova; + struct page **pages; +}; + +/* + * These define the different GMU wake up options - these define how both the + * CPU and the GMU bring up the hardware + */ + +/* THe GMU has already been booted and the rentention registers are active */ +#define GMU_WARM_BOOT 0 + +/* the GMU is coming up for the first time or back from a power collapse */ +#define GMU_COLD_BOOT 1 + +/* The GMU is being soft reset after a fault */ +#define GMU_RESET 2 + +/* + * These define the level of control that the GMU has - the higher the number + * the more things that the GMU hardware controls on its own. + */ + +/* The GMU does not do any idle state management */ +#define GMU_IDLE_STATE_ACTIVE 0 + +/* The GMU manages SPTP power collapse */ +#define GMU_IDLE_STATE_SPTP 2 + +/* The GMU does automatic IFPC (intra-frame power collapse) */ +#define GMU_IDLE_STATE_IFPC 3 + +struct a6xx_gmu { + struct device *dev; + + void * __iomem mmio; + void * __iomem pdc_mmio; + + int hfi_irq; + int gmu_irq; + + struct regulator *gx; + + struct iommu_domain *domain; + u64 uncached_iova_base; + + int idle_level; + + struct a6xx_gmu_bo *hfi; + struct a6xx_gmu_bo *debug; + + int nr_clocks; + struct clk_bulk_data *clocks; + struct clk *core_clk; + + int nr_gpu_freqs; + unsigned long gpu_freqs[16]; + u32 gx_arc_votes[16]; + + int nr_gmu_freqs; + unsigned long gmu_freqs[4]; + u32 cx_arc_votes[4]; + + struct a6xx_hfi_queue queues[2]; + + struct tasklet_struct hfi_tasklet; +}; + +static inline u32 gmu_read(struct a6xx_gmu *gmu, u32 offset) +{ + return msm_readl(gmu->mmio + (offset << 2)); +} + +static inline void gmu_write(struct a6xx_gmu *gmu, u32 offset, u32 value) +{ + return msm_writel(value, gmu->mmio + (offset << 2)); +} + +static inline void pdc_write(struct a6xx_gmu *gmu, u32 offset, u32 value) +{ + return msm_writel(value, gmu->pdc_mmio + (offset << 2)); +} + +static inline void gmu_rmw(struct a6xx_gmu *gmu, u32 reg, u32 mask, u32 or) +{ + u32 val = gmu_read(gmu, reg); + + val &= ~mask; + + gmu_write(gmu, reg, val | or); +} + +#define gmu_poll_timeout(gmu, addr, val, cond, interval, timeout) \ + readl_poll_timeout((gmu)->mmio + ((addr) << 2), val, cond, \ + interval, timeout) + +/* + * These are the available OOB (out of band requests) to the GMU where "out of + * band" means that the CPU talks to the GMU directly and not through HFI. + * Normally this works by writing a ITCM/DTCM register and then triggering a + * interrupt (the "request" bit) and waiting for an acknowledgment (the "ack" + * bit). The state is cleared by writing the "clear' bit to the GMU interrupt. + * + * These are used to force the GMU/GPU to stay on during a critical sequence or + * for hardware workarounds. + */ + +enum a6xx_gmu_oob_state { + GMU_OOB_BOOT_SLUMBER = 0, + GMU_OOB_GPU_SET, + GMU_OOB_DCVS_SET, +}; + +/* These are the interrupt / ack bits for each OOB request that are set + * in a6xx_gmu_set_oob and a6xx_clear_oob + */ + +/* + * Let the GMU know that a boot or slumber operation has started. The value in + * REG_A6XX_GMU_BOOT_SLUMBER_OPTION lets the GMU know which operation we are + * doing + */ +#define GMU_OOB_BOOT_SLUMBER_REQUEST 22 +#define GMU_OOB_BOOT_SLUMBER_ACK 30 +#define GMU_OOB_BOOT_SLUMBER_CLEAR 30 + +/* + * Set a new power level for the GPU when the CPU is doing frequency scaling + */ +#define GMU_OOB_DCVS_REQUEST 23 +#define GMU_OOB_DCVS_ACK 31 +#define GMU_OOB_DCVS_CLEAR 31 + +/* + * Let the GMU know to not turn off any GPU registers while the CPU is in a + * critical section + */ +#define GMU_OOB_GPU_SET_REQUEST 16 +#define GMU_OOB_GPU_SET_ACK 24 +#define GMU_OOB_GPU_SET_CLEAR 24 + + +void a6xx_hfi_init(struct a6xx_gmu *gmu); +int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state); +void a6xx_hfi_stop(struct a6xx_gmu *gmu); + +void a6xx_hfi_task(unsigned long data); + +#endif diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h new file mode 100644 index 000000000000..ef68098d2adc --- /dev/null +++ b/drivers/gpu/drm/msm/adreno/a6xx_gmu.xml.h @@ -0,0 +1,382 @@ +#ifndef A6XX_GMU_XML +#define A6XX_GMU_XML + +/* Autogenerated file, DO NOT EDIT manually! + +This file was generated by the rules-ng-ng headergen tool in this git repository: +http://github.com/freedreno/envytools/ +git clone https://github.com/freedreno/envytools.git + +The rules-ng-ng source files this header was generated from are: +- /home/robclark/src/envytools/rnndb/adreno.xml ( 501 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml ( 36805 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml ( 13634 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml ( 42393 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml ( 112086 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml ( 147240 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml ( 101627 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml ( 10431 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: +- Rob Clark <robdclark@gmail.com> (robclark) +- Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice (including the +next paragraph) shall be included in all copies or substantial +portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +*/ + + +#define A6XX_GMU_GPU_IDLE_STATUS_BUSY_IGN_AHB 0x00800000 +#define A6XX_GMU_GPU_IDLE_STATUS_CX_GX_CPU_BUSY_IGN_AHB 0x40000000 +#define A6XX_GMU_OOB_BOOT_SLUMBER_SET_MASK 0x00400000 +#define A6XX_GMU_OOB_BOOT_SLUMBER_CHECK_MASK 0x40000000 +#define A6XX_GMU_OOB_BOOT_SLUMBER_CLEAR_MASK 0x40000000 +#define A6XX_GMU_OOB_DCVS_SET_MASK 0x00800000 +#define A6XX_GMU_OOB_DCVS_CHECK_MASK 0x80000000 +#define A6XX_GMU_OOB_DCVS_CLEAR_MASK 0x80000000 +#define A6XX_GMU_OOB_GPU_SET_MASK 0x00040000 +#define A6XX_GMU_OOB_GPU_CHECK_MASK 0x04000000 +#define A6XX_GMU_OOB_GPU_CLEAR_MASK 0x04000000 +#define A6XX_GMU_OOB_PERFCNTR_SET_MASK 0x00020000 +#define A6XX_GMU_OOB_PERFCNTR_CHECK_MASK 0x02000000 +#define A6XX_GMU_OOB_PERFCNTR_CLEAR_MASK 0x02000000 +#define A6XX_HFI_IRQ_MSGQ_MASK 0x00000001 +#define A6XX_HFI_IRQ_DSGQ_MASK 0x00000002 +#define A6XX_HFI_IRQ_BLOCKED_MSG_MASK 0x00000004 +#define A6XX_HFI_IRQ_CM3_FAULT_MASK 0x00800000 +#define A6XX_HFI_IRQ_GMU_ERR_MASK__MASK 0x007f0000 +#define A6XX_HFI_IRQ_GMU_ERR_MASK__SHIFT 16 +static inline uint32_t A6XX_HFI_IRQ_GMU_ERR_MASK(uint32_t val) +{ + return ((val) << A6XX_HFI_IRQ_GMU_ERR_MASK__SHIFT) & A6XX_HFI_IRQ_GMU_ERR_MASK__MASK; +} +#define A6XX_HFI_IRQ_OOB_MASK__MASK 0xff000000 +#define A6XX_HFI_IRQ_OOB_MASK__SHIFT 24 +static inline uint32_t A6XX_HFI_IRQ_OOB_MASK(uint32_t val) +{ + return ((val) << A6XX_HFI_IRQ_OOB_MASK__SHIFT) & A6XX_HFI_IRQ_OOB_MASK__MASK; +} +#define A6XX_HFI_H2F_IRQ_MASK_BIT 0x00000001 +#define REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL 0x00000080 + +#define REG_A6XX_GMU_GX_SPTPRAC_POWER_CONTROL 0x00000081 + +#define REG_A6XX_GMU_CM3_ITCM_START 0x00000c00 + +#define REG_A6XX_GMU_CM3_DTCM_START 0x00001c00 + +#define REG_A6XX_GMU_NMI_CONTROL_STATUS 0x000023f0 + +#define REG_A6XX_GMU_BOOT_SLUMBER_OPTION 0x000023f8 + +#define REG_A6XX_GMU_GX_VOTE_IDX 0x000023f9 + +#define REG_A6XX_GMU_MX_VOTE_IDX 0x000023fa + +#define REG_A6XX_GMU_DCVS_ACK_OPTION 0x000023fc + +#define REG_A6XX_GMU_DCVS_PERF_SETTING 0x000023fd + +#define REG_A6XX_GMU_DCVS_BW_SETTING 0x000023fe + +#define REG_A6XX_GMU_DCVS_RETURN 0x000023ff + +#define REG_A6XX_GMU_SYS_BUS_CONFIG 0x00004c0f + +#define REG_A6XX_GMU_CM3_SYSRESET 0x00005000 + +#define REG_A6XX_GMU_CM3_BOOT_CONFIG 0x00005001 + +#define REG_A6XX_GMU_CM3_FW_BUSY 0x0000501a + +#define REG_A6XX_GMU_CM3_FW_INIT_RESULT 0x0000501c + +#define REG_A6XX_GMU_CM3_CFG 0x0000502d + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE 0x00005040 + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0 0x00005041 + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_1 0x00005042 + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_L 0x00005044 + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_0_H 0x00005045 + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_L 0x00005046 + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_1_H 0x00005047 + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_L 0x00005048 + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_2_H 0x00005049 + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_L 0x0000504a + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_3_H 0x0000504b + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_L 0x0000504c + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_4_H 0x0000504d + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_L 0x0000504e + +#define REG_A6XX_GMU_CX_GMU_POWER_COUNTER_XOCLK_5_H 0x0000504f + +#define REG_A6XX_GMU_PWR_COL_INTER_FRAME_CTRL 0x000050c0 +#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_IFPC_ENABLE 0x00000001 +#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_HM_POWER_COLLAPSE_ENABLE 0x00000002 +#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_SPTPRAC_POWER_CONTROL_ENABLE 0x00000004 +#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__MASK 0x00003c00 +#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__SHIFT 10 +static inline uint32_t A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS(uint32_t val) +{ + return ((val) << A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__SHIFT) & A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_NUM_PASS_SKIPS__MASK; +} +#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__MASK 0xffffc000 +#define A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__SHIFT 14 +static inline uint32_t A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH(uint32_t val) +{ + return ((val) << A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__SHIFT) & A6XX_GMU_PWR_COL_INTER_FRAME_CTRL_MIN_PASS_LENGTH__MASK; +} + +#define REG_A6XX_GMU_PWR_COL_INTER_FRAME_HYST 0x000050c1 + +#define REG_A6XX_GMU_PWR_COL_SPTPRAC_HYST 0x000050c2 + +#define REG_A6XX_GMU_SPTPRAC_PWR_CLK_STATUS 0x000050d0 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWERING_OFF 0x00000001 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWERING_ON 0x00000002 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_ON 0x00000004 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SPTPRAC_GDSC_POWER_OFF 0x00000008 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_SP_CLOCK_OFF 0x00000010 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GMU_UP_POWER_STATE 0x00000020 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_GDSC_POWER_OFF 0x00000040 +#define A6XX_GMU_SPTPRAC_PWR_CLK_STATUS_GX_HM_CLK_OFF 0x00000080 + +#define REG_A6XX_GMU_GPU_NAP_CTRL 0x000050e4 +#define A6XX_GMU_GPU_NAP_CTRL_HW_NAP_ENABLE 0x00000001 +#define A6XX_GMU_GPU_NAP_CTRL_SID__MASK 0x000001f0 +#define A6XX_GMU_GPU_NAP_CTRL_SID__SHIFT 4 +static inline uint32_t A6XX_GMU_GPU_NAP_CTRL_SID(uint32_t val) +{ + return ((val) << A6XX_GMU_GPU_NAP_CTRL_SID__SHIFT) & A6XX_GMU_GPU_NAP_CTRL_SID__MASK; +} + +#define REG_A6XX_GMU_RPMH_CTRL 0x000050e8 +#define A6XX_GMU_RPMH_CTRL_RPMH_INTERFACE_ENABLE 0x00000001 +#define A6XX_GMU_RPMH_CTRL_LLC_VOTE_ENABLE 0x00000010 +#define A6XX_GMU_RPMH_CTRL_DDR_VOTE_ENABLE 0x00000100 +#define A6XX_GMU_RPMH_CTRL_MX_VOTE_ENABLE 0x00000200 +#define A6XX_GMU_RPMH_CTRL_CX_VOTE_ENABLE 0x00000400 +#define A6XX_GMU_RPMH_CTRL_GFX_VOTE_ENABLE 0x00000800 +#define A6XX_GMU_RPMH_CTRL_DDR_MIN_VOTE_ENABLE 0x00001000 +#define A6XX_GMU_RPMH_CTRL_MX_MIN_VOTE_ENABLE 0x00002000 +#define A6XX_GMU_RPMH_CTRL_CX_MIN_VOTE_ENABLE 0x00004000 +#define A6XX_GMU_RPMH_CTRL_GFX_MIN_VOTE_ENABLE 0x00008000 + +#define REG_A6XX_GMU_RPMH_HYST_CTRL 0x000050e9 + +#define REG_A6XX_GPU_GMU_CX_GMU_RPMH_POWER_STATE 0x000050ec + +#define REG_A6XX_GMU_BOOT_KMD_LM_HANDSHAKE 0x000051f0 + +#define REG_A6XX_GMU_LLM_GLM_SLEEP_CTRL 0x00005157 + +#define REG_A6XX_GMU_LLM_GLM_SLEEP_STATUS 0x00005158 + +#define REG_A6XX_GMU_ALWAYS_ON_COUNTER_L 0x00005088 + +#define REG_A6XX_GMU_ALWAYS_ON_COUNTER_H 0x00005089 + +#define REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE 0x000050c3 + +#define REG_A6XX_GMU_HFI_CTRL_STATUS 0x00005180 + +#define REG_A6XX_GMU_HFI_VERSION_INFO 0x00005181 + +#define REG_A6XX_GMU_HFI_SFR_ADDR 0x00005182 + +#define REG_A6XX_GMU_HFI_MMAP_ADDR 0x00005183 + +#define REG_A6XX_GMU_HFI_QTBL_INFO 0x00005184 + +#define REG_A6XX_GMU_HFI_QTBL_ADDR 0x00005185 + +#define REG_A6XX_GMU_HFI_CTRL_INIT 0x00005186 + +#define REG_A6XX_GMU_GMU2HOST_INTR_SET 0x00005190 + +#define REG_A6XX_GMU_GMU2HOST_INTR_CLR 0x00005191 + +#define REG_A6XX_GMU_GMU2HOST_INTR_INFO 0x00005192 +#define A6XX_GMU_GMU2HOST_INTR_INFO_MSGQ 0x00000001 +#define A6XX_GMU_GMU2HOST_INTR_INFO_CM3_FAULT 0x00800000 + +#define REG_A6XX_GMU_GMU2HOST_INTR_MASK 0x00005193 + +#define REG_A6XX_GMU_HOST2GMU_INTR_SET 0x00005194 + +#define REG_A6XX_GMU_HOST2GMU_INTR_CLR 0x00005195 + +#define REG_A6XX_GMU_HOST2GMU_INTR_RAW_INFO 0x00005196 + +#define REG_A6XX_GMU_HOST2GMU_INTR_EN_0 0x00005197 + +#define REG_A6XX_GMU_HOST2GMU_INTR_EN_1 0x00005198 + +#define REG_A6XX_GMU_HOST2GMU_INTR_EN_2 0x00005199 + +#define REG_A6XX_GMU_HOST2GMU_INTR_EN_3 0x0000519a + +#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_0 0x0000519b + +#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_1 0x0000519c + +#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_2 0x0000519d + +#define REG_A6XX_GMU_HOST2GMU_INTR_INFO_3 0x0000519e + +#define REG_A6XX_GMU_GENERAL_1 0x000051c6 + +#define REG_A6XX_GMU_GENERAL_7 0x000051cc + +#define REG_A6XX_GMU_ISENSE_CTRL 0x0000515d + +#define REG_A6XX_GPU_CS_ENABLE_REG 0x00008920 + +#define REG_A6XX_GPU_GMU_CX_GMU_ISENSE_CTRL 0x0000515d + +#define REG_A6XX_GPU_CS_AMP_CALIBRATION_CONTROL3 0x00008578 + +#define REG_A6XX_GPU_CS_AMP_CALIBRATION_CONTROL2 0x00008558 + +#define REG_A6XX_GPU_CS_A_SENSOR_CTRL_0 0x00008580 + +#define REG_A6XX_GPU_CS_A_SENSOR_CTRL_2 0x00027ada + +#define REG_A6XX_GPU_CS_SENSOR_GENERAL_STATUS 0x0000881a + +#define REG_A6XX_GPU_CS_AMP_CALIBRATION_CONTROL1 0x00008957 + +#define REG_A6XX_GPU_CS_SENSOR_GENERAL_STATUS 0x0000881a + +#define REG_A6XX_GPU_CS_AMP_CALIBRATION_STATUS1_0 0x0000881d + +#define REG_A6XX_GPU_CS_AMP_CALIBRATION_STATUS1_2 0x0000881f + +#define REG_A6XX_GPU_CS_AMP_CALIBRATION_STATUS1_4 0x00008821 + +#define REG_A6XX_GPU_CS_AMP_CALIBRATION_DONE 0x00008965 + +#define REG_A6XX_GPU_CS_AMP_PERIOD_CTRL 0x0000896d + +#define REG_A6XX_GPU_CS_AMP_CALIBRATION_DONE 0x00008965 + +#define REG_A6XX_GPU_GMU_CX_GMU_PWR_THRESHOLD 0x0000514d + +#define REG_A6XX_GMU_AO_INTERRUPT_EN 0x00009303 + +#define REG_A6XX_GMU_AO_HOST_INTERRUPT_CLR 0x00009304 + +#define REG_A6XX_GMU_AO_HOST_INTERRUPT_STATUS 0x00009305 +#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_WDOG_BITE 0x00000001 +#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_RSCC_COMP 0x00000002 +#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_VDROOP 0x00000004 +#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_FENCE_ERR 0x00000008 +#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_DBD_WAKEUP 0x00000010 +#define A6XX_GMU_AO_HOST_INTERRUPT_STATUS_HOST_AHB_BUS_ERROR 0x00000020 + +#define REG_A6XX_GMU_AO_HOST_INTERRUPT_MASK 0x00009306 + +#define REG_A6XX_GPU_GMU_AO_GMU_CGC_MODE_CNTL 0x00009309 + +#define REG_A6XX_GPU_GMU_AO_GMU_CGC_DELAY_CNTL 0x0000930a + +#define REG_A6XX_GPU_GMU_AO_GMU_CGC_HYST_CNTL 0x0000930b + +#define REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS 0x0000930c +#define A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS_GPUBUSYIGNAHB 0x00800000 + +#define REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_STATUS2 0x0000930d + +#define REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK 0x0000930e + +#define REG_A6XX_GMU_AO_AHB_FENCE_CTRL 0x00009310 + +#define REG_A6XX_GMU_AHB_FENCE_STATUS 0x00009313 + +#define REG_A6XX_GMU_RBBM_INT_UNMASKED_STATUS 0x00009315 + +#define REG_A6XX_GMU_AO_SPARE_CNTL 0x00009316 + +#define REG_A6XX_GPU_RSCC_RSC_STATUS0_DRV0 0x00008c04 + +#define REG_A6XX_GMU_RSCC_CONTROL_REQ 0x00009307 + +#define REG_A6XX_GMU_RSCC_CONTROL_ACK 0x00009308 + +#define REG_A6XX_GMU_AHB_FENCE_RANGE_0 0x00009311 + +#define REG_A6XX_GMU_AHB_FENCE_RANGE_1 0x00009312 + +#define REG_A6XX_GPU_CC_GX_GDSCR 0x00009c03 + +#define REG_A6XX_GPU_CC_GX_DOMAIN_MISC 0x00009d42 + +#define REG_A6XX_RSCC_PDC_SEQ_START_ADDR 0x00008c08 + +#define REG_A6XX_RSCC_PDC_MATCH_VALUE_LO 0x00008c09 + +#define REG_A6XX_RSCC_PDC_MATCH_VALUE_HI 0x00008c0a + +#define REG_A6XX_RSCC_PDC_SLAVE_ID_DRV0 0x00008c0b + +#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_ADDR 0x00008c0d + +#define REG_A6XX_RSCC_HIDDEN_TCS_CMD0_DATA 0x00008c0e + +#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_L_DRV0 0x00008c82 + +#define REG_A6XX_RSCC_TIMESTAMP_UNIT0_TIMESTAMP_H_DRV0 0x00008c83 + +#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_EN_DRV0 0x00008c89 + +#define REG_A6XX_RSCC_TIMESTAMP_UNIT1_OUTPUT_DRV0 0x00008c8c + +#define REG_A6XX_RSCC_OVERRIDE_START_ADDR 0x00008d00 + +#define REG_A6XX_RSCC_SEQ_BUSY_DRV0 0x00008d01 + +#define REG_A6XX_RSCC_SEQ_MEM_0_DRV0 0x00008d80 + +#define REG_A6XX_RSCC_TCS0_DRV0_STATUS 0x00008f46 + +#define REG_A6XX_RSCC_TCS1_DRV0_STATUS 0x000090ae + +#define REG_A6XX_RSCC_TCS2_DRV0_STATUS 0x00009216 + +#define REG_A6XX_RSCC_TCS3_DRV0_STATUS 0x0000937e + + +#endif /* A6XX_GMU_XML */ diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.c b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c new file mode 100644 index 000000000000..c629f742a1d1 --- /dev/null +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.c @@ -0,0 +1,818 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */ + + +#include "msm_gem.h" +#include "msm_mmu.h" +#include "a6xx_gpu.h" +#include "a6xx_gmu.xml.h" + +static inline bool _a6xx_check_idle(struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + + /* Check that the GMU is idle */ + if (!a6xx_gmu_isidle(&a6xx_gpu->gmu)) + return false; + + /* Check tha the CX master is idle */ + if (gpu_read(gpu, REG_A6XX_RBBM_STATUS) & + ~A6XX_RBBM_STATUS_CP_AHB_BUSY_CX_MASTER) + return false; + + return !(gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS) & + A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT); +} + +bool a6xx_idle(struct msm_gpu *gpu, struct msm_ringbuffer *ring) +{ + /* wait for CP to drain ringbuffer: */ + if (!adreno_idle(gpu, ring)) + return false; + + if (spin_until(_a6xx_check_idle(gpu))) { + DRM_ERROR("%s: %ps: timeout waiting for GPU to idle: status %8.8X irq %8.8X rptr/wptr %d/%d\n", + gpu->name, __builtin_return_address(0), + gpu_read(gpu, REG_A6XX_RBBM_STATUS), + gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS), + gpu_read(gpu, REG_A6XX_CP_RB_RPTR), + gpu_read(gpu, REG_A6XX_CP_RB_WPTR)); + return false; + } + + return true; +} + +static void a6xx_flush(struct msm_gpu *gpu, struct msm_ringbuffer *ring) +{ + uint32_t wptr; + unsigned long flags; + + spin_lock_irqsave(&ring->lock, flags); + + /* Copy the shadow to the actual register */ + ring->cur = ring->next; + + /* Make sure to wrap wptr if we need to */ + wptr = get_wptr(ring); + + spin_unlock_irqrestore(&ring->lock, flags); + + /* Make sure everything is posted before making a decision */ + mb(); + + gpu_write(gpu, REG_A6XX_CP_RB_WPTR, wptr); +} + +static void a6xx_submit(struct msm_gpu *gpu, struct msm_gem_submit *submit, + struct msm_file_private *ctx) +{ + struct msm_drm_private *priv = gpu->dev->dev_private; + struct msm_ringbuffer *ring = submit->ring; + unsigned int i; + + /* Invalidate CCU depth and color */ + OUT_PKT7(ring, CP_EVENT_WRITE, 1); + OUT_RING(ring, PC_CCU_INVALIDATE_DEPTH); + + OUT_PKT7(ring, CP_EVENT_WRITE, 1); + OUT_RING(ring, PC_CCU_INVALIDATE_COLOR); + + /* Submit the commands */ + for (i = 0; i < submit->nr_cmds; i++) { + switch (submit->cmd[i].type) { + case MSM_SUBMIT_CMD_IB_TARGET_BUF: + break; + case MSM_SUBMIT_CMD_CTX_RESTORE_BUF: + if (priv->lastctx == ctx) + break; + case MSM_SUBMIT_CMD_BUF: + OUT_PKT7(ring, CP_INDIRECT_BUFFER_PFE, 3); + OUT_RING(ring, lower_32_bits(submit->cmd[i].iova)); + OUT_RING(ring, upper_32_bits(submit->cmd[i].iova)); + OUT_RING(ring, submit->cmd[i].size); + break; + } + } + + /* Write the fence to the scratch register */ + OUT_PKT4(ring, REG_A6XX_CP_SCRATCH_REG(2), 1); + OUT_RING(ring, submit->seqno); + + /* + * Execute a CACHE_FLUSH_TS event. This will ensure that the + * timestamp is written to the memory and then triggers the interrupt + */ + OUT_PKT7(ring, CP_EVENT_WRITE, 4); + OUT_RING(ring, CACHE_FLUSH_TS | (1 << 31)); + OUT_RING(ring, lower_32_bits(rbmemptr(ring, fence))); + OUT_RING(ring, upper_32_bits(rbmemptr(ring, fence))); + OUT_RING(ring, submit->seqno); + + a6xx_flush(gpu, ring); +} + +static const struct { + u32 offset; + u32 value; +} a6xx_hwcg[] = { + {REG_A6XX_RBBM_CLOCK_CNTL_SP0, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL_SP1, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL_SP2, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL_SP3, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_SP0, 0x02022220}, + {REG_A6XX_RBBM_CLOCK_CNTL2_SP1, 0x02022220}, + {REG_A6XX_RBBM_CLOCK_CNTL2_SP2, 0x02022220}, + {REG_A6XX_RBBM_CLOCK_CNTL2_SP3, 0x02022220}, + {REG_A6XX_RBBM_CLOCK_DELAY_SP0, 0x00000080}, + {REG_A6XX_RBBM_CLOCK_DELAY_SP1, 0x00000080}, + {REG_A6XX_RBBM_CLOCK_DELAY_SP2, 0x00000080}, + {REG_A6XX_RBBM_CLOCK_DELAY_SP3, 0x00000080}, + {REG_A6XX_RBBM_CLOCK_HYST_SP0, 0x0000f3cf}, + {REG_A6XX_RBBM_CLOCK_HYST_SP1, 0x0000f3cf}, + {REG_A6XX_RBBM_CLOCK_HYST_SP2, 0x0000f3cf}, + {REG_A6XX_RBBM_CLOCK_HYST_SP3, 0x0000f3cf}, + {REG_A6XX_RBBM_CLOCK_CNTL_TP0, 0x02222222}, + {REG_A6XX_RBBM_CLOCK_CNTL_TP1, 0x02222222}, + {REG_A6XX_RBBM_CLOCK_CNTL_TP2, 0x02222222}, + {REG_A6XX_RBBM_CLOCK_CNTL_TP3, 0x02222222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_TP0, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_TP1, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_TP2, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_TP3, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL3_TP0, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL3_TP1, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL3_TP2, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL3_TP3, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL4_TP0, 0x00022222}, + {REG_A6XX_RBBM_CLOCK_CNTL4_TP1, 0x00022222}, + {REG_A6XX_RBBM_CLOCK_CNTL4_TP2, 0x00022222}, + {REG_A6XX_RBBM_CLOCK_CNTL4_TP3, 0x00022222}, + {REG_A6XX_RBBM_CLOCK_HYST_TP0, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST_TP1, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST_TP2, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST_TP3, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST2_TP0, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST2_TP1, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST2_TP2, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST2_TP3, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST3_TP0, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST3_TP1, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST3_TP2, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST3_TP3, 0x77777777}, + {REG_A6XX_RBBM_CLOCK_HYST4_TP0, 0x00077777}, + {REG_A6XX_RBBM_CLOCK_HYST4_TP1, 0x00077777}, + {REG_A6XX_RBBM_CLOCK_HYST4_TP2, 0x00077777}, + {REG_A6XX_RBBM_CLOCK_HYST4_TP3, 0x00077777}, + {REG_A6XX_RBBM_CLOCK_DELAY_TP0, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY_TP1, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY_TP2, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY_TP3, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY2_TP0, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY2_TP1, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY2_TP2, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY2_TP3, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY3_TP0, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY3_TP1, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY3_TP2, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY3_TP3, 0x11111111}, + {REG_A6XX_RBBM_CLOCK_DELAY4_TP0, 0x00011111}, + {REG_A6XX_RBBM_CLOCK_DELAY4_TP1, 0x00011111}, + {REG_A6XX_RBBM_CLOCK_DELAY4_TP2, 0x00011111}, + {REG_A6XX_RBBM_CLOCK_DELAY4_TP3, 0x00011111}, + {REG_A6XX_RBBM_CLOCK_CNTL_UCHE, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_UCHE, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL3_UCHE, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL4_UCHE, 0x00222222}, + {REG_A6XX_RBBM_CLOCK_HYST_UCHE, 0x00000004}, + {REG_A6XX_RBBM_CLOCK_DELAY_UCHE, 0x00000002}, + {REG_A6XX_RBBM_CLOCK_CNTL_RB0, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL_RB1, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL_RB2, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL_RB3, 0x22222222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_RB0, 0x00002222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_RB1, 0x00002222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_RB2, 0x00002222}, + {REG_A6XX_RBBM_CLOCK_CNTL2_RB3, 0x00002222}, + {REG_A6XX_RBBM_CLOCK_CNTL_CCU0, 0x00002220}, + {REG_A6XX_RBBM_CLOCK_CNTL_CCU1, 0x00002220}, + {REG_A6XX_RBBM_CLOCK_CNTL_CCU2, 0x00002220}, + {REG_A6XX_RBBM_CLOCK_CNTL_CCU3, 0x00002220}, + {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU0, 0x00040f00}, + {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU1, 0x00040f00}, + {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU2, 0x00040f00}, + {REG_A6XX_RBBM_CLOCK_HYST_RB_CCU3, 0x00040f00}, + {REG_A6XX_RBBM_CLOCK_CNTL_RAC, 0x05022022}, + {REG_A6XX_RBBM_CLOCK_CNTL2_RAC, 0x00005555}, + {REG_A6XX_RBBM_CLOCK_DELAY_RAC, 0x00000011}, + {REG_A6XX_RBBM_CLOCK_HYST_RAC, 0x00445044}, + {REG_A6XX_RBBM_CLOCK_CNTL_TSE_RAS_RBBM, 0x04222222}, + {REG_A6XX_RBBM_CLOCK_MODE_GPC, 0x00222222}, + {REG_A6XX_RBBM_CLOCK_MODE_VFD, 0x00002222}, + {REG_A6XX_RBBM_CLOCK_HYST_TSE_RAS_RBBM, 0x00000000}, + {REG_A6XX_RBBM_CLOCK_HYST_GPC, 0x04104004}, + {REG_A6XX_RBBM_CLOCK_HYST_VFD, 0x00000000}, + {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ, 0x00000000}, + {REG_A6XX_RBBM_CLOCK_DELAY_TSE_RAS_RBBM, 0x00004000}, + {REG_A6XX_RBBM_CLOCK_DELAY_GPC, 0x00000200}, + {REG_A6XX_RBBM_CLOCK_DELAY_VFD, 0x00002222}, + {REG_A6XX_RBBM_CLOCK_DELAY_HLSQ_2, 0x00000002}, + {REG_A6XX_RBBM_CLOCK_MODE_HLSQ, 0x00002222}, + {REG_A6XX_RBBM_CLOCK_CNTL_GMU_GX, 0x00000222}, + {REG_A6XX_RBBM_CLOCK_DELAY_GMU_GX, 0x00000111}, + {REG_A6XX_RBBM_CLOCK_HYST_GMU_GX, 0x00000555} +}; + +static void a6xx_set_hwcg(struct msm_gpu *gpu, bool state) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + struct a6xx_gmu *gmu = &a6xx_gpu->gmu; + unsigned int i; + u32 val; + + val = gpu_read(gpu, REG_A6XX_RBBM_CLOCK_CNTL); + + /* Don't re-program the registers if they are already correct */ + if ((!state && !val) || (state && (val == 0x8aa8aa02))) + return; + + /* Disable SP clock before programming HWCG registers */ + gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 1, 0); + + for (i = 0; i < ARRAY_SIZE(a6xx_hwcg); i++) + gpu_write(gpu, a6xx_hwcg[i].offset, + state ? a6xx_hwcg[i].value : 0); + + /* Enable SP clock */ + gmu_rmw(gmu, REG_A6XX_GPU_GMU_GX_SPTPRAC_CLOCK_CONTROL, 0, 1); + + gpu_write(gpu, REG_A6XX_RBBM_CLOCK_CNTL, state ? 0x8aa8aa02 : 0); +} + +static int a6xx_cp_init(struct msm_gpu *gpu) +{ + struct msm_ringbuffer *ring = gpu->rb[0]; + + OUT_PKT7(ring, CP_ME_INIT, 8); + + OUT_RING(ring, 0x0000002f); + + /* Enable multiple hardware contexts */ + OUT_RING(ring, 0x00000003); + + /* Enable error detection */ + OUT_RING(ring, 0x20000000); + + /* Don't enable header dump */ + OUT_RING(ring, 0x00000000); + OUT_RING(ring, 0x00000000); + + /* No workarounds enabled */ + OUT_RING(ring, 0x00000000); + + /* Pad rest of the cmds with 0's */ + OUT_RING(ring, 0x00000000); + OUT_RING(ring, 0x00000000); + + a6xx_flush(gpu, ring); + return a6xx_idle(gpu, ring) ? 0 : -EINVAL; +} + +static int a6xx_ucode_init(struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + + if (!a6xx_gpu->sqe_bo) { + a6xx_gpu->sqe_bo = adreno_fw_create_bo(gpu, + adreno_gpu->fw[ADRENO_FW_SQE], &a6xx_gpu->sqe_iova); + + if (IS_ERR(a6xx_gpu->sqe_bo)) { + int ret = PTR_ERR(a6xx_gpu->sqe_bo); + + a6xx_gpu->sqe_bo = NULL; + DRM_DEV_ERROR(&gpu->pdev->dev, + "Could not allocate SQE ucode: %d\n", ret); + + return ret; + } + } + + gpu_write64(gpu, REG_A6XX_CP_SQE_INSTR_BASE_LO, + REG_A6XX_CP_SQE_INSTR_BASE_HI, a6xx_gpu->sqe_iova); + + return 0; +} + +#define A6XX_INT_MASK (A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR | \ + A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW | \ + A6XX_RBBM_INT_0_MASK_CP_HW_ERROR | \ + A6XX_RBBM_INT_0_MASK_CP_IB2 | \ + A6XX_RBBM_INT_0_MASK_CP_IB1 | \ + A6XX_RBBM_INT_0_MASK_CP_RB | \ + A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS | \ + A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW | \ + A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT | \ + A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS | \ + A6XX_RBBM_INT_0_MASK_UCHE_TRAP_INTR) + +static int a6xx_hw_init(struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + int ret; + + /* Make sure the GMU keeps the GPU on while we set it up */ + a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + + gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_CNTL, 0); + + /* + * Disable the trusted memory range - we don't actually supported secure + * memory rendering at this point in time and we don't want to block off + * part of the virtual memory space. + */ + gpu_write64(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_LO, + REG_A6XX_RBBM_SECVID_TSB_TRUSTED_BASE_HI, 0x00000000); + gpu_write(gpu, REG_A6XX_RBBM_SECVID_TSB_TRUSTED_SIZE, 0x00000000); + + /* enable hardware clockgating */ + a6xx_set_hwcg(gpu, true); + + /* VBIF start */ + gpu_write(gpu, REG_A6XX_VBIF_GATE_OFF_WRREQ_EN, 0x00000009); + gpu_write(gpu, REG_A6XX_RBBM_VBIF_CLIENT_QOS_CNTL, 0x3); + + /* Make all blocks contribute to the GPU BUSY perf counter */ + gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_GPU_BUSY_MASKED, 0xffffffff); + + /* Disable L2 bypass in the UCHE */ + gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_LO, 0xffffffc0); + gpu_write(gpu, REG_A6XX_UCHE_WRITE_RANGE_MAX_HI, 0x0001ffff); + gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_LO, 0xfffff000); + gpu_write(gpu, REG_A6XX_UCHE_TRAP_BASE_HI, 0x0001ffff); + gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_LO, 0xfffff000); + gpu_write(gpu, REG_A6XX_UCHE_WRITE_THRU_BASE_HI, 0x0001ffff); + + /* Set the GMEM VA range [0x100000:0x100000 + gpu->gmem - 1] */ + gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MIN_LO, + REG_A6XX_UCHE_GMEM_RANGE_MIN_HI, 0x00100000); + + gpu_write64(gpu, REG_A6XX_UCHE_GMEM_RANGE_MAX_LO, + REG_A6XX_UCHE_GMEM_RANGE_MAX_HI, + 0x00100000 + adreno_gpu->gmem - 1); + + gpu_write(gpu, REG_A6XX_UCHE_FILTER_CNTL, 0x804); + gpu_write(gpu, REG_A6XX_UCHE_CACHE_WAYS, 0x4); + + gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_2, 0x010000c0); + gpu_write(gpu, REG_A6XX_CP_ROQ_THRESHOLDS_1, 0x8040362c); + + /* Setting the mem pool size */ + gpu_write(gpu, REG_A6XX_CP_MEM_POOL_SIZE, 128); + + /* Setting the primFifo thresholds default values */ + gpu_write(gpu, REG_A6XX_PC_DBG_ECO_CNTL, (0x300 << 11)); + + /* Set the AHB default slave response to "ERROR" */ + gpu_write(gpu, REG_A6XX_CP_AHB_CNTL, 0x1); + + /* Turn on performance counters */ + gpu_write(gpu, REG_A6XX_RBBM_PERFCTR_CNTL, 0x1); + + /* Select CP0 to always count cycles */ + gpu_write(gpu, REG_A6XX_CP_PERFCTR_CP_SEL_0, PERF_CP_ALWAYS_COUNT); + + /* FIXME: not sure if this should live here or in a6xx_gmu.c */ + gmu_write(&a6xx_gpu->gmu, REG_A6XX_GPU_GMU_AO_GPU_CX_BUSY_MASK, + 0xff000000); + gmu_rmw(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_SELECT_0, + 0xff, 0x20); + gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_CX_GMU_POWER_COUNTER_ENABLE, + 0x01); + + gpu_write(gpu, REG_A6XX_RB_NC_MODE_CNTL, 2 << 1); + gpu_write(gpu, REG_A6XX_TPL1_NC_MODE_CNTL, 2 << 1); + gpu_write(gpu, REG_A6XX_SP_NC_MODE_CNTL, 2 << 1); + gpu_write(gpu, REG_A6XX_UCHE_MODE_CNTL, 2 << 21); + + /* Enable fault detection */ + gpu_write(gpu, REG_A6XX_RBBM_INTERFACE_HANG_INT_CNTL, + (1 << 30) | 0x1fffff); + + gpu_write(gpu, REG_A6XX_UCHE_CLIENT_PF, 1); + + /* Protect registers from the CP */ + gpu_write(gpu, REG_A6XX_CP_PROTECT_CNTL, 0x00000003); + + gpu_write(gpu, REG_A6XX_CP_PROTECT(0), + A6XX_PROTECT_RDONLY(0x600, 0x51)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(1), A6XX_PROTECT_RW(0xae50, 0x2)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(2), A6XX_PROTECT_RW(0x9624, 0x13)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(3), A6XX_PROTECT_RW(0x8630, 0x8)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(4), A6XX_PROTECT_RW(0x9e70, 0x1)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(5), A6XX_PROTECT_RW(0x9e78, 0x187)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(6), A6XX_PROTECT_RW(0xf000, 0x810)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(7), + A6XX_PROTECT_RDONLY(0xfc00, 0x3)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(8), A6XX_PROTECT_RW(0x50e, 0x0)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(9), A6XX_PROTECT_RDONLY(0x50f, 0x0)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(10), A6XX_PROTECT_RW(0x510, 0x0)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(11), + A6XX_PROTECT_RDONLY(0x0, 0x4f9)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(12), + A6XX_PROTECT_RDONLY(0x501, 0xa)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(13), + A6XX_PROTECT_RDONLY(0x511, 0x44)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(14), A6XX_PROTECT_RW(0xe00, 0xe)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(15), A6XX_PROTECT_RW(0x8e00, 0x0)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(16), A6XX_PROTECT_RW(0x8e50, 0xf)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(17), A6XX_PROTECT_RW(0xbe02, 0x0)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(18), + A6XX_PROTECT_RW(0xbe20, 0x11f3)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(19), A6XX_PROTECT_RW(0x800, 0x82)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(20), A6XX_PROTECT_RW(0x8a0, 0x8)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(21), A6XX_PROTECT_RW(0x8ab, 0x19)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(22), A6XX_PROTECT_RW(0x900, 0x4d)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(23), A6XX_PROTECT_RW(0x98d, 0x76)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(24), + A6XX_PROTECT_RDONLY(0x8d0, 0x23)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(25), + A6XX_PROTECT_RDONLY(0x980, 0x4)); + gpu_write(gpu, REG_A6XX_CP_PROTECT(26), A6XX_PROTECT_RW(0xa630, 0x0)); + + /* Enable interrupts */ + gpu_write(gpu, REG_A6XX_RBBM_INT_0_MASK, A6XX_INT_MASK); + + ret = adreno_hw_init(gpu); + if (ret) + goto out; + + ret = a6xx_ucode_init(gpu); + if (ret) + goto out; + + /* Always come up on rb 0 */ + a6xx_gpu->cur_ring = gpu->rb[0]; + + /* Enable the SQE_to start the CP engine */ + gpu_write(gpu, REG_A6XX_CP_SQE_CNTL, 1); + + ret = a6xx_cp_init(gpu); + if (ret) + goto out; + + gpu_write(gpu, REG_A6XX_RBBM_SECVID_TRUST_CNTL, 0x0); + +out: + /* + * Tell the GMU that we are done touching the GPU and it can start power + * management + */ + a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + + /* Take the GMU out of its special boot mode */ + a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_BOOT_SLUMBER); + + return ret; +} + +static void a6xx_dump(struct msm_gpu *gpu) +{ + dev_info(&gpu->pdev->dev, "status: %08x\n", + gpu_read(gpu, REG_A6XX_RBBM_STATUS)); + adreno_dump(gpu); +} + +#define VBIF_RESET_ACK_TIMEOUT 100 +#define VBIF_RESET_ACK_MASK 0x00f0 + +static void a6xx_recover(struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + int i; + + adreno_dump_info(gpu); + + for (i = 0; i < 8; i++) + dev_info(&gpu->pdev->dev, "CP_SCRATCH_REG%d: %u\n", i, + gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(i))); + + if (hang_debug) + a6xx_dump(gpu); + + /* + * Turn off keep alive that might have been enabled by the hang + * interrupt + */ + gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 0); + + gpu->funcs->pm_suspend(gpu); + gpu->funcs->pm_resume(gpu); + + msm_gpu_hw_init(gpu); +} + +static int a6xx_fault_handler(void *arg, unsigned long iova, int flags) +{ + struct msm_gpu *gpu = arg; + + pr_warn_ratelimited("*** gpu fault: iova=%08lx, flags=%d (%u,%u,%u,%u)\n", + iova, flags, + gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(4)), + gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(5)), + gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(6)), + gpu_read(gpu, REG_A6XX_CP_SCRATCH_REG(7))); + + return -EFAULT; +} + +static void a6xx_cp_hw_err_irq(struct msm_gpu *gpu) +{ + u32 status = gpu_read(gpu, REG_A6XX_CP_INTERRUPT_STATUS); + + if (status & A6XX_CP_INT_CP_OPCODE_ERROR) { + u32 val; + + gpu_write(gpu, REG_A6XX_CP_SQE_STAT_ADDR, 1); + val = gpu_read(gpu, REG_A6XX_CP_SQE_STAT_DATA); + dev_err_ratelimited(&gpu->pdev->dev, + "CP | opcode error | possible opcode=0x%8.8X\n", + val); + } + + if (status & A6XX_CP_INT_CP_UCODE_ERROR) + dev_err_ratelimited(&gpu->pdev->dev, + "CP ucode error interrupt\n"); + + if (status & A6XX_CP_INT_CP_HW_FAULT_ERROR) + dev_err_ratelimited(&gpu->pdev->dev, "CP | HW fault | status=0x%8.8X\n", + gpu_read(gpu, REG_A6XX_CP_HW_FAULT)); + + if (status & A6XX_CP_INT_CP_REGISTER_PROTECTION_ERROR) { + u32 val = gpu_read(gpu, REG_A6XX_CP_PROTECT_STATUS); + + dev_err_ratelimited(&gpu->pdev->dev, + "CP | protected mode error | %s | addr=0x%8.8X | status=0x%8.8X\n", + val & (1 << 20) ? "READ" : "WRITE", + (val & 0x3ffff), val); + } + + if (status & A6XX_CP_INT_CP_AHB_ERROR) + dev_err_ratelimited(&gpu->pdev->dev, "CP AHB error interrupt\n"); + + if (status & A6XX_CP_INT_CP_VSD_PARITY_ERROR) + dev_err_ratelimited(&gpu->pdev->dev, "CP VSD decoder parity error\n"); + + if (status & A6XX_CP_INT_CP_ILLEGAL_INSTR_ERROR) + dev_err_ratelimited(&gpu->pdev->dev, "CP illegal instruction error\n"); + +} + +static void a6xx_fault_detect_irq(struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + struct drm_device *dev = gpu->dev; + struct msm_drm_private *priv = dev->dev_private; + struct msm_ringbuffer *ring = gpu->funcs->active_ring(gpu); + + /* + * Force the GPU to stay on until after we finish + * collecting information + */ + gmu_write(&a6xx_gpu->gmu, REG_A6XX_GMU_GMU_PWR_COL_KEEPALIVE, 1); + + DRM_DEV_ERROR(&gpu->pdev->dev, + "gpu fault ring %d fence %x status %8.8X rb %4.4x/%4.4x ib1 %16.16llX/%4.4x ib2 %16.16llX/%4.4x\n", + ring ? ring->id : -1, ring ? ring->seqno : 0, + gpu_read(gpu, REG_A6XX_RBBM_STATUS), + gpu_read(gpu, REG_A6XX_CP_RB_RPTR), + gpu_read(gpu, REG_A6XX_CP_RB_WPTR), + gpu_read64(gpu, REG_A6XX_CP_IB1_BASE, REG_A6XX_CP_IB1_BASE_HI), + gpu_read(gpu, REG_A6XX_CP_IB1_REM_SIZE), + gpu_read64(gpu, REG_A6XX_CP_IB2_BASE, REG_A6XX_CP_IB2_BASE_HI), + gpu_read(gpu, REG_A6XX_CP_IB2_REM_SIZE)); + + /* Turn off the hangcheck timer to keep it from bothering us */ + del_timer(&gpu->hangcheck_timer); + + queue_work(priv->wq, &gpu->recover_work); +} + +static irqreturn_t a6xx_irq(struct msm_gpu *gpu) +{ + u32 status = gpu_read(gpu, REG_A6XX_RBBM_INT_0_STATUS); + + gpu_write(gpu, REG_A6XX_RBBM_INT_CLEAR_CMD, status); + + if (status & A6XX_RBBM_INT_0_MASK_RBBM_HANG_DETECT) + a6xx_fault_detect_irq(gpu); + + if (status & A6XX_RBBM_INT_0_MASK_CP_AHB_ERROR) + dev_err_ratelimited(&gpu->pdev->dev, "CP | AHB bus error\n"); + + if (status & A6XX_RBBM_INT_0_MASK_CP_HW_ERROR) + a6xx_cp_hw_err_irq(gpu); + + if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_ASYNCFIFO_OVERFLOW) + dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB ASYNC overflow\n"); + + if (status & A6XX_RBBM_INT_0_MASK_RBBM_ATB_BUS_OVERFLOW) + dev_err_ratelimited(&gpu->pdev->dev, "RBBM | ATB bus overflow\n"); + + if (status & A6XX_RBBM_INT_0_MASK_UCHE_OOB_ACCESS) + dev_err_ratelimited(&gpu->pdev->dev, "UCHE | Out of bounds access\n"); + + if (status & A6XX_RBBM_INT_0_MASK_CP_CACHE_FLUSH_TS) + msm_gpu_retire(gpu); + + return IRQ_HANDLED; +} + +static const u32 a6xx_register_offsets[REG_ADRENO_REGISTER_MAX] = { + REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE, REG_A6XX_CP_RB_BASE), + REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_BASE_HI, REG_A6XX_CP_RB_BASE_HI), + REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR, + REG_A6XX_CP_RB_RPTR_ADDR_LO), + REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR_ADDR_HI, + REG_A6XX_CP_RB_RPTR_ADDR_HI), + REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_RPTR, REG_A6XX_CP_RB_RPTR), + REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_WPTR, REG_A6XX_CP_RB_WPTR), + REG_ADRENO_DEFINE(REG_ADRENO_CP_RB_CNTL, REG_A6XX_CP_RB_CNTL), +}; + +static const u32 a6xx_registers[] = { + 0x0000, 0x0002, 0x0010, 0x0010, 0x0012, 0x0012, 0x0018, 0x001b, + 0x001e, 0x0032, 0x0038, 0x003c, 0x0042, 0x0042, 0x0044, 0x0044, + 0x0047, 0x0047, 0x0056, 0x0056, 0x00ad, 0x00ae, 0x00b0, 0x00fb, + 0x0100, 0x011d, 0x0200, 0x020d, 0x0210, 0x0213, 0x0218, 0x023d, + 0x0400, 0x04f9, 0x0500, 0x0500, 0x0505, 0x050b, 0x050e, 0x0511, + 0x0533, 0x0533, 0x0540, 0x0555, 0x0800, 0x0808, 0x0810, 0x0813, + 0x0820, 0x0821, 0x0823, 0x0827, 0x0830, 0x0833, 0x0840, 0x0843, + 0x084f, 0x086f, 0x0880, 0x088a, 0x08a0, 0x08ab, 0x08c0, 0x08c4, + 0x08d0, 0x08dd, 0x08f0, 0x08f3, 0x0900, 0x0903, 0x0908, 0x0911, + 0x0928, 0x093e, 0x0942, 0x094d, 0x0980, 0x0984, 0x098d, 0x0996, + 0x0998, 0x099e, 0x09a0, 0x09a6, 0x09a8, 0x09ae, 0x09b0, 0x09b1, + 0x09c2, 0x09c8, 0x0a00, 0x0a03, 0x0c00, 0x0c04, 0x0c06, 0x0c06, + 0x0c10, 0x0cd9, 0x0e00, 0x0e0e, 0x0e10, 0x0e13, 0x0e17, 0x0e19, + 0x0e1c, 0x0e2b, 0x0e30, 0x0e32, 0x0e38, 0x0e39, 0x8600, 0x8601, + 0x8610, 0x861b, 0x8620, 0x8620, 0x8628, 0x862b, 0x8630, 0x8637, + 0x8e01, 0x8e01, 0x8e04, 0x8e05, 0x8e07, 0x8e08, 0x8e0c, 0x8e0c, + 0x8e10, 0x8e1c, 0x8e20, 0x8e25, 0x8e28, 0x8e28, 0x8e2c, 0x8e2f, + 0x8e3b, 0x8e3e, 0x8e40, 0x8e43, 0x8e50, 0x8e5e, 0x8e70, 0x8e77, + 0x9600, 0x9604, 0x9624, 0x9637, 0x9e00, 0x9e01, 0x9e03, 0x9e0e, + 0x9e11, 0x9e16, 0x9e19, 0x9e19, 0x9e1c, 0x9e1c, 0x9e20, 0x9e23, + 0x9e30, 0x9e31, 0x9e34, 0x9e34, 0x9e70, 0x9e72, 0x9e78, 0x9e79, + 0x9e80, 0x9fff, 0xa600, 0xa601, 0xa603, 0xa603, 0xa60a, 0xa60a, + 0xa610, 0xa617, 0xa630, 0xa630, + ~0 +}; + +static int a6xx_pm_resume(struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + int ret; + + ret = a6xx_gmu_resume(a6xx_gpu); + + gpu->needs_hw_init = true; + + return ret; +} + +static int a6xx_pm_suspend(struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + + /* + * Make sure the GMU is idle before continuing (because some transitions + * may use VBIF + */ + a6xx_gmu_wait_for_idle(a6xx_gpu); + + /* Clear the VBIF pipe before shutting down */ + /* FIXME: This accesses the GPU - do we need to make sure it is on? */ + gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0xf); + spin_until((gpu_read(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL1) & 0xf) == 0xf); + gpu_write(gpu, REG_A6XX_VBIF_XIN_HALT_CTRL0, 0); + + return a6xx_gmu_stop(a6xx_gpu); +} + +static int a6xx_get_timestamp(struct msm_gpu *gpu, uint64_t *value) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + + /* Force the GPU power on so we can read this register */ + a6xx_gmu_set_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + + *value = gpu_read64(gpu, REG_A6XX_RBBM_PERFCTR_CP_0_LO, + REG_A6XX_RBBM_PERFCTR_CP_0_HI); + + a6xx_gmu_clear_oob(&a6xx_gpu->gmu, GMU_OOB_GPU_SET); + return 0; +} + +#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) +static void a6xx_show(struct msm_gpu *gpu, struct msm_gpu_state *state, + struct drm_printer *p) +{ + adreno_show(gpu, state, p); +} +#endif + +static struct msm_ringbuffer *a6xx_active_ring(struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + + return a6xx_gpu->cur_ring; +} + +static void a6xx_destroy(struct msm_gpu *gpu) +{ + struct adreno_gpu *adreno_gpu = to_adreno_gpu(gpu); + struct a6xx_gpu *a6xx_gpu = to_a6xx_gpu(adreno_gpu); + + if (a6xx_gpu->sqe_bo) { + if (a6xx_gpu->sqe_iova) + msm_gem_put_iova(a6xx_gpu->sqe_bo, gpu->aspace); + drm_gem_object_unreference_unlocked(a6xx_gpu->sqe_bo); + } + + a6xx_gmu_remove(a6xx_gpu); + + adreno_gpu_cleanup(adreno_gpu); + kfree(a6xx_gpu); +} + +static const struct adreno_gpu_funcs funcs = { + .base = { + .get_param = adreno_get_param, + .hw_init = a6xx_hw_init, + .pm_suspend = a6xx_pm_suspend, + .pm_resume = a6xx_pm_resume, + .recover = a6xx_recover, + .submit = a6xx_submit, + .flush = a6xx_flush, + .active_ring = a6xx_active_ring, + .irq = a6xx_irq, + .destroy = a6xx_destroy, +#if defined(CONFIG_DEBUG_FS) || defined(CONFIG_DEV_COREDUMP) + .show = a6xx_show, +#endif + }, + .get_timestamp = a6xx_get_timestamp, +}; + +struct msm_gpu *a6xx_gpu_init(struct drm_device *dev) +{ + struct msm_drm_private *priv = dev->dev_private; + struct platform_device *pdev = priv->gpu_pdev; + struct device_node *node; + struct a6xx_gpu *a6xx_gpu; + struct adreno_gpu *adreno_gpu; + struct msm_gpu *gpu; + int ret; + + a6xx_gpu = kzalloc(sizeof(*a6xx_gpu), GFP_KERNEL); + if (!a6xx_gpu) + return ERR_PTR(-ENOMEM); + + adreno_gpu = &a6xx_gpu->base; + gpu = &adreno_gpu->base; + + adreno_gpu->registers = a6xx_registers; + adreno_gpu->reg_offsets = a6xx_register_offsets; + + ret = adreno_gpu_init(dev, pdev, adreno_gpu, &funcs, 1); + if (ret) { + a6xx_destroy(&(a6xx_gpu->base.base)); + return ERR_PTR(ret); + } + + /* Check if there is a GMU phandle and set it up */ + node = of_parse_phandle(pdev->dev.of_node, "gmu", 0); + + /* FIXME: How do we gracefully handle this? */ + BUG_ON(!node); + + ret = a6xx_gmu_probe(a6xx_gpu, node); + if (ret) { + a6xx_destroy(&(a6xx_gpu->base.base)); + return ERR_PTR(ret); + } + + if (gpu->aspace) + msm_mmu_set_fault_handler(gpu->aspace->mmu, gpu, + a6xx_fault_handler); + + return gpu; +} diff --git a/drivers/gpu/drm/msm/adreno/a6xx_gpu.h b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h new file mode 100644 index 000000000000..dd69e5b0e692 --- /dev/null +++ b/drivers/gpu/drm/msm/adreno/a6xx_gpu.h @@ -0,0 +1,60 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2017 The Linux Foundation. All rights reserved. */ + +#ifndef __A6XX_GPU_H__ +#define __A6XX_GPU_H__ + + +#include "adreno_gpu.h" +#include "a6xx.xml.h" + +#include "a6xx_gmu.h" + +extern bool hang_debug; + +struct a6xx_gpu { + struct adreno_gpu base; + + struct drm_gem_object *sqe_bo; + uint64_t sqe_iova; + + struct msm_ringbuffer *cur_ring; + + struct a6xx_gmu gmu; +}; + +#define to_a6xx_gpu(x) container_of(x, struct a6xx_gpu, base) + +/* + * Given a register and a count, return a value to program into + * REG_CP_PROTECT_REG(n) - this will block both reads and writes for _len + * registers starting at _reg. + */ +#define A6XX_PROTECT_RW(_reg, _len) \ + ((1 << 31) | \ + (((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF)) + +/* + * Same as above, but allow reads over the range. For areas of mixed use (such + * as performance counters) this allows us to protect a much larger range with a + * single register + */ +#define A6XX_PROTECT_RDONLY(_reg, _len) \ + ((((_len) & 0x3FFF) << 18) | ((_reg) & 0x3FFFF)) + + +int a6xx_gmu_resume(struct a6xx_gpu *gpu); +int a6xx_gmu_stop(struct a6xx_gpu *gpu); + +int a6xx_gmu_wait_for_idle(struct a6xx_gpu *gpu); + +int a6xx_gmu_reset(struct a6xx_gpu *a6xx_gpu); +bool a6xx_gmu_isidle(struct a6xx_gmu *gmu); + +int a6xx_gmu_set_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state); +void a6xx_gmu_clear_oob(struct a6xx_gmu *gmu, enum a6xx_gmu_oob_state state); + +int a6xx_gmu_probe(struct a6xx_gpu *a6xx_gpu, struct device_node *node); +void a6xx_gmu_remove(struct a6xx_gpu *a6xx_gpu); + +#endif /* __A6XX_GPU_H__ */ diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.c b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c new file mode 100644 index 000000000000..f19ef4cb6ea4 --- /dev/null +++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.c @@ -0,0 +1,435 @@ +// SPDX-License-Identifier: GPL-2.0 +/* Copyright (c) 2017-2018 The Linux Foundation. All rights reserved. */ + +#include <linux/completion.h> +#include <linux/circ_buf.h> +#include <linux/list.h> + +#include "a6xx_gmu.h" +#include "a6xx_gmu.xml.h" + +#define HFI_MSG_ID(val) [val] = #val + +static const char * const a6xx_hfi_msg_id[] = { + HFI_MSG_ID(HFI_H2F_MSG_INIT), + HFI_MSG_ID(HFI_H2F_MSG_FW_VERSION), + HFI_MSG_ID(HFI_H2F_MSG_BW_TABLE), + HFI_MSG_ID(HFI_H2F_MSG_PERF_TABLE), + HFI_MSG_ID(HFI_H2F_MSG_TEST), +}; + +static int a6xx_hfi_queue_read(struct a6xx_hfi_queue *queue, u32 *data, + u32 dwords) +{ + struct a6xx_hfi_queue_header *header = queue->header; + u32 i, hdr, index = header->read_index; + + if (header->read_index == header->write_index) { + header->rx_request = 1; + return 0; + } + + hdr = queue->data[index]; + + /* + * If we are to assume that the GMU firmware is in fact a rational actor + * and is programmed to not send us a larger response than we expect + * then we can also assume that if the header size is unexpectedly large + * that it is due to memory corruption and/or hardware failure. In this + * case the only reasonable course of action is to BUG() to help harden + * the failure. + */ + + BUG_ON(HFI_HEADER_SIZE(hdr) > dwords); + + for (i = 0; i < HFI_HEADER_SIZE(hdr); i++) { + data[i] = queue->data[index]; + index = (index + 1) % header->size; + } + + header->read_index = index; + return HFI_HEADER_SIZE(hdr); +} + +static int a6xx_hfi_queue_write(struct a6xx_gmu *gmu, + struct a6xx_hfi_queue *queue, u32 *data, u32 dwords) +{ + struct a6xx_hfi_queue_header *header = queue->header; + u32 i, space, index = header->write_index; + + spin_lock(&queue->lock); + + space = CIRC_SPACE(header->write_index, header->read_index, + header->size); + if (space < dwords) { + header->dropped++; + spin_unlock(&queue->lock); + return -ENOSPC; + } + + for (i = 0; i < dwords; i++) { + queue->data[index] = data[i]; + index = (index + 1) % header->size; + } + + header->write_index = index; + spin_unlock(&queue->lock); + + gmu_write(gmu, REG_A6XX_GMU_HOST2GMU_INTR_SET, 0x01); + return 0; +} + +struct a6xx_hfi_response { + u32 id; + u32 seqnum; + struct list_head node; + struct completion complete; + + u32 error; + u32 payload[16]; +}; + +/* + * Incoming HFI ack messages can come in out of order so we need to store all + * the pending messages on a list until they are handled. + */ +static spinlock_t hfi_ack_lock = __SPIN_LOCK_UNLOCKED(message_lock); +static LIST_HEAD(hfi_ack_list); + +static void a6xx_hfi_handle_ack(struct a6xx_gmu *gmu, + struct a6xx_hfi_msg_response *msg) +{ + struct a6xx_hfi_response *resp; + u32 id, seqnum; + + /* msg->ret_header contains the header of the message being acked */ + id = HFI_HEADER_ID(msg->ret_header); + seqnum = HFI_HEADER_SEQNUM(msg->ret_header); + + spin_lock(&hfi_ack_lock); + list_for_each_entry(resp, &hfi_ack_list, node) { + if (resp->id == id && resp->seqnum == seqnum) { + resp->error = msg->error; + memcpy(resp->payload, msg->payload, + sizeof(resp->payload)); + + complete(&resp->complete); + spin_unlock(&hfi_ack_lock); + return; + } + } + spin_unlock(&hfi_ack_lock); + + dev_err(gmu->dev, "Nobody was waiting for HFI message %d\n", seqnum); +} + +static void a6xx_hfi_handle_error(struct a6xx_gmu *gmu, + struct a6xx_hfi_msg_response *msg) +{ + struct a6xx_hfi_msg_error *error = (struct a6xx_hfi_msg_error *) msg; + + dev_err(gmu->dev, "GMU firmware error %d\n", error->code); +} + +void a6xx_hfi_task(unsigned long data) +{ + struct a6xx_gmu *gmu = (struct a6xx_gmu *) data; + struct a6xx_hfi_queue *queue = &gmu->queues[HFI_RESPONSE_QUEUE]; + struct a6xx_hfi_msg_response resp; + + for (;;) { + u32 id; + int ret = a6xx_hfi_queue_read(queue, (u32 *) &resp, + sizeof(resp) >> 2); + + /* Returns the number of bytes copied or negative on error */ + if (ret <= 0) { + if (ret < 0) + dev_err(gmu->dev, + "Unable to read the HFI message queue\n"); + break; + } + + id = HFI_HEADER_ID(resp.header); + + if (id == HFI_F2H_MSG_ACK) + a6xx_hfi_handle_ack(gmu, &resp); + else if (id == HFI_F2H_MSG_ERROR) + a6xx_hfi_handle_error(gmu, &resp); + } +} + +static int a6xx_hfi_send_msg(struct a6xx_gmu *gmu, int id, + void *data, u32 size, u32 *payload, u32 payload_size) +{ + struct a6xx_hfi_queue *queue = &gmu->queues[HFI_COMMAND_QUEUE]; + struct a6xx_hfi_response resp = { 0 }; + int ret, dwords = size >> 2; + u32 seqnum; + + seqnum = atomic_inc_return(&queue->seqnum) % 0xfff; + + /* First dword of the message is the message header - fill it in */ + *((u32 *) data) = (seqnum << 20) | (HFI_MSG_CMD << 16) | + (dwords << 8) | id; + + init_completion(&resp.complete); + resp.id = id; + resp.seqnum = seqnum; + + spin_lock_bh(&hfi_ack_lock); + list_add_tail(&resp.node, &hfi_ack_list); + spin_unlock_bh(&hfi_ack_lock); + + ret = a6xx_hfi_queue_write(gmu, queue, data, dwords); + if (ret) { + dev_err(gmu->dev, "Unable to send message %s id %d\n", + a6xx_hfi_msg_id[id], seqnum); + goto out; + } + + /* Wait up to 5 seconds for the response */ + ret = wait_for_completion_timeout(&resp.complete, + msecs_to_jiffies(5000)); + if (!ret) { + dev_err(gmu->dev, + "Message %s id %d timed out waiting for response\n", + a6xx_hfi_msg_id[id], seqnum); + ret = -ETIMEDOUT; + } else + ret = 0; + +out: + spin_lock_bh(&hfi_ack_lock); + list_del(&resp.node); + spin_unlock_bh(&hfi_ack_lock); + + if (ret) + return ret; + + if (resp.error) { + dev_err(gmu->dev, "Message %s id %d returned error %d\n", + a6xx_hfi_msg_id[id], seqnum, resp.error); + return -EINVAL; + } + + if (payload && payload_size) { + int copy = min_t(u32, payload_size, sizeof(resp.payload)); + + memcpy(payload, resp.payload, copy); + } + + return 0; +} + +static int a6xx_hfi_send_gmu_init(struct a6xx_gmu *gmu, int boot_state) +{ + struct a6xx_hfi_msg_gmu_init_cmd msg = { 0 }; + + msg.dbg_buffer_addr = (u32) gmu->debug->iova; + msg.dbg_buffer_size = (u32) gmu->debug->size; + msg.boot_state = boot_state; + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_INIT, &msg, sizeof(msg), + NULL, 0); +} + +static int a6xx_hfi_get_fw_version(struct a6xx_gmu *gmu, u32 *version) +{ + struct a6xx_hfi_msg_fw_version msg = { 0 }; + + /* Currently supporting version 1.1 */ + msg.supported_version = (1 << 28) | (1 << 16); + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_FW_VERSION, &msg, sizeof(msg), + version, sizeof(*version)); +} + +static int a6xx_hfi_send_perf_table(struct a6xx_gmu *gmu) +{ + struct a6xx_hfi_msg_perf_table msg = { 0 }; + int i; + + msg.num_gpu_levels = gmu->nr_gpu_freqs; + msg.num_gmu_levels = gmu->nr_gmu_freqs; + + for (i = 0; i < gmu->nr_gpu_freqs; i++) { + msg.gx_votes[i].vote = gmu->gx_arc_votes[i]; + msg.gx_votes[i].freq = gmu->gpu_freqs[i] / 1000; + } + + for (i = 0; i < gmu->nr_gmu_freqs; i++) { + msg.cx_votes[i].vote = gmu->cx_arc_votes[i]; + msg.cx_votes[i].freq = gmu->gmu_freqs[i] / 1000; + } + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_PERF_TABLE, &msg, sizeof(msg), + NULL, 0); +} + +static int a6xx_hfi_send_bw_table(struct a6xx_gmu *gmu) +{ + struct a6xx_hfi_msg_bw_table msg = { 0 }; + + /* + * The sdm845 GMU doesn't do bus frequency scaling on its own but it + * does need at least one entry in the list because it might be accessed + * when the GMU is shutting down. Send a single "off" entry. + */ + + msg.bw_level_num = 1; + + msg.ddr_cmds_num = 3; + msg.ddr_wait_bitmask = 0x07; + + msg.ddr_cmds_addrs[0] = 0x50000; + msg.ddr_cmds_addrs[1] = 0x5005c; + msg.ddr_cmds_addrs[2] = 0x5000c; + + msg.ddr_cmds_data[0][0] = 0x40000000; + msg.ddr_cmds_data[0][1] = 0x40000000; + msg.ddr_cmds_data[0][2] = 0x40000000; + + /* + * These are the CX (CNOC) votes. This is used but the values for the + * sdm845 GMU are known and fixed so we can hard code them. + */ + + msg.cnoc_cmds_num = 3; + msg.cnoc_wait_bitmask = 0x05; + + msg.cnoc_cmds_addrs[0] = 0x50034; + msg.cnoc_cmds_addrs[1] = 0x5007c; + msg.cnoc_cmds_addrs[2] = 0x5004c; + + msg.cnoc_cmds_data[0][0] = 0x40000000; + msg.cnoc_cmds_data[0][1] = 0x00000000; + msg.cnoc_cmds_data[0][2] = 0x40000000; + + msg.cnoc_cmds_data[1][0] = 0x60000001; + msg.cnoc_cmds_data[1][1] = 0x20000001; + msg.cnoc_cmds_data[1][2] = 0x60000001; + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_BW_TABLE, &msg, sizeof(msg), + NULL, 0); +} + +static int a6xx_hfi_send_test(struct a6xx_gmu *gmu) +{ + struct a6xx_hfi_msg_test msg = { 0 }; + + return a6xx_hfi_send_msg(gmu, HFI_H2F_MSG_TEST, &msg, sizeof(msg), + NULL, 0); +} + +int a6xx_hfi_start(struct a6xx_gmu *gmu, int boot_state) +{ + int ret; + + ret = a6xx_hfi_send_gmu_init(gmu, boot_state); + if (ret) + return ret; + + ret = a6xx_hfi_get_fw_version(gmu, NULL); + if (ret) + return ret; + + /* + * We have to get exchange version numbers per the sequence but at this + * point th kernel driver doesn't need to know the exact version of + * the GMU firmware + */ + + ret = a6xx_hfi_send_perf_table(gmu); + if (ret) + return ret; + + ret = a6xx_hfi_send_bw_table(gmu); + if (ret) + return ret; + + /* + * Let the GMU know that there won't be any more HFI messages until next + * boot + */ + a6xx_hfi_send_test(gmu); + + return 0; +} + +void a6xx_hfi_stop(struct a6xx_gmu *gmu) +{ + int i; + + for (i = 0; i < ARRAY_SIZE(gmu->queues); i++) { + struct a6xx_hfi_queue *queue = &gmu->queues[i]; + + if (!queue->header) + continue; + + if (queue->header->read_index != queue->header->write_index) + dev_err(gmu->dev, "HFI queue %d is not empty\n", i); + + queue->header->read_index = 0; + queue->header->write_index = 0; + } +} + +static void a6xx_hfi_queue_init(struct a6xx_hfi_queue *queue, + struct a6xx_hfi_queue_header *header, void *virt, u64 iova, + u32 id) +{ + spin_lock_init(&queue->lock); + queue->header = header; + queue->data = virt; + atomic_set(&queue->seqnum, 0); + + /* Set up the shared memory header */ + header->iova = iova; + header->type = 10 << 8 | id; + header->status = 1; + header->size = SZ_4K >> 2; + header->msg_size = 0; + header->dropped = 0; + header->rx_watermark = 1; + header->tx_watermark = 1; + header->rx_request = 1; + header->tx_request = 0; + header->read_index = 0; + header->write_index = 0; +} + +void a6xx_hfi_init(struct a6xx_gmu *gmu) +{ + struct a6xx_gmu_bo *hfi = gmu->hfi; + struct a6xx_hfi_queue_table_header *table = hfi->virt; + struct a6xx_hfi_queue_header *headers = hfi->virt + sizeof(*table); + u64 offset; + int table_size; + + /* + * The table size is the size of the table header plus all of the queue + * headers + */ + table_size = sizeof(*table); + table_size += (ARRAY_SIZE(gmu->queues) * + sizeof(struct a6xx_hfi_queue_header)); + + table->version = 0; + table->size = table_size; + /* First queue header is located immediately after the table header */ + table->qhdr0_offset = sizeof(*table) >> 2; + table->qhdr_size = sizeof(struct a6xx_hfi_queue_header) >> 2; + table->num_queues = ARRAY_SIZE(gmu->queues); + table->active_queues = ARRAY_SIZE(gmu->queues); + + /* Command queue */ + offset = SZ_4K; + a6xx_hfi_queue_init(&gmu->queues[0], &headers[0], hfi->virt + offset, + hfi->iova + offset, 0); + + /* GMU response queue */ + offset += SZ_4K; + a6xx_hfi_queue_init(&gmu->queues[1], &headers[1], hfi->virt + offset, + hfi->iova + offset, 4); +} diff --git a/drivers/gpu/drm/msm/adreno/a6xx_hfi.h b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h new file mode 100644 index 000000000000..60d1319fa44f --- /dev/null +++ b/drivers/gpu/drm/msm/adreno/a6xx_hfi.h @@ -0,0 +1,127 @@ +/* SPDX-License-Identifier: GPL-2.0 */ +/* Copyright (c) 2017 The Linux Foundation. All rights reserved. */ + +#ifndef _A6XX_HFI_H_ +#define _A6XX_HFI_H_ + +struct a6xx_hfi_queue_table_header { + u32 version; + u32 size; /* Size of the queue table in dwords */ + u32 qhdr0_offset; /* Offset of the first queue header */ + u32 qhdr_size; /* Size of the queue headers */ + u32 num_queues; /* Number of total queues */ + u32 active_queues; /* Number of active queues */ +}; + +struct a6xx_hfi_queue_header { + u32 status; + u32 iova; + u32 type; + u32 size; + u32 msg_size; + u32 dropped; + u32 rx_watermark; + u32 tx_watermark; + u32 rx_request; + u32 tx_request; + u32 read_index; + u32 write_index; +}; + +struct a6xx_hfi_queue { + struct a6xx_hfi_queue_header *header; + spinlock_t lock; + u32 *data; + atomic_t seqnum; +}; + +/* This is the outgoing queue to the GMU */ +#define HFI_COMMAND_QUEUE 0 + +/* THis is the incoming response queue from the GMU */ +#define HFI_RESPONSE_QUEUE 1 + +#define HFI_HEADER_ID(msg) ((msg) & 0xff) +#define HFI_HEADER_SIZE(msg) (((msg) >> 8) & 0xff) +#define HFI_HEADER_SEQNUM(msg) (((msg) >> 20) & 0xfff) + +/* FIXME: Do we need this or can we use ARRAY_SIZE? */ +#define HFI_RESPONSE_PAYLOAD_SIZE 16 + +/* HFI message types */ + +#define HFI_MSG_CMD 0 +#define HFI_MSG_ACK 2 + +#define HFI_F2H_MSG_ACK 126 + +struct a6xx_hfi_msg_response { + u32 header; + u32 ret_header; + u32 error; + u32 payload[HFI_RESPONSE_PAYLOAD_SIZE]; +}; + +#define HFI_F2H_MSG_ERROR 100 + +struct a6xx_hfi_msg_error { + u32 header; + u32 code; + u32 payload[2]; +}; + +#define HFI_H2F_MSG_INIT 0 + +struct a6xx_hfi_msg_gmu_init_cmd { + u32 header; + u32 seg_id; + u32 dbg_buffer_addr; + u32 dbg_buffer_size; + u32 boot_state; +}; + +#define HFI_H2F_MSG_FW_VERSION 1 + +struct a6xx_hfi_msg_fw_version { + u32 header; + u32 supported_version; +}; + +#define HFI_H2F_MSG_PERF_TABLE 4 + +struct perf_level { + u32 vote; + u32 freq; +}; + +struct a6xx_hfi_msg_perf_table { + u32 header; + u32 num_gpu_levels; + u32 num_gmu_levels; + + struct perf_level gx_votes[16]; + struct perf_level cx_votes[4]; +}; + +#define HFI_H2F_MSG_BW_TABLE 3 + +struct a6xx_hfi_msg_bw_table { + u32 header; + u32 bw_level_num; + u32 cnoc_cmds_num; + u32 ddr_cmds_num; + u32 cnoc_wait_bitmask; + u32 ddr_wait_bitmask; + u32 cnoc_cmds_addrs[6]; + u32 cnoc_cmds_data[2][6]; + u32 ddr_cmds_addrs[8]; + u32 ddr_cmds_data[16][8]; +}; + +#define HFI_H2F_MSG_TEST 5 + +struct a6xx_hfi_msg_test { + u32 header; +}; + +#endif diff --git a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h index b634cf71352b..5dace1350810 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_common.xml.h +++ b/drivers/gpu/drm/msm/adreno/adreno_common.xml.h @@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 431 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 37162 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 13324 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 31866 bytes, from 2017-06-06 18:26:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 111898 bytes, from 2017-06-06 18:23:59) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml ( 139480 bytes, from 2017-06-16 12:44:39) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/adreno.xml ( 501 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml ( 36805 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml ( 13634 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml ( 42393 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml ( 112086 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml ( 147240 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml ( 101627 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml ( 10431 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) @@ -44,6 +46,14 @@ WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. */ +enum chip { + A2XX = 0, + A3XX = 0, + A4XX = 0, + A5XX = 0, + A6XX = 0, +}; + enum adreno_pa_su_sc_draw { PC_DRAW_POINTS = 0, PC_DRAW_LINES = 1, @@ -181,6 +191,12 @@ enum a3xx_rb_blend_opcode { BLEND_MAX_DST_SRC = 4, }; +enum a4xx_tess_spacing { + EQUAL_SPACING = 0, + ODD_SPACING = 2, + EVEN_SPACING = 3, +}; + #define REG_AXXX_CP_RB_BASE 0x000001c0 #define REG_AXXX_CP_RB_CNTL 0x000001c1 diff --git a/drivers/gpu/drm/msm/adreno/adreno_device.c b/drivers/gpu/drm/msm/adreno/adreno_device.c index 44813624a286..7d3e9a129ac7 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_device.c +++ b/drivers/gpu/drm/msm/adreno/adreno_device.c @@ -111,6 +111,16 @@ static const struct adreno_info gpulist[] = { ADRENO_QUIRK_FAULT_DETECT_MASK, .init = a5xx_gpu_init, .zapfw = "a530_zap.mdt", + }, { + .rev = ADRENO_REV(6, 3, 0, ANY_ID), + .revn = 630, + .name = "A630", + .fw = { + [ADRENO_FW_SQE] = "a630_sqe.fw", + [ADRENO_FW_GMU] = "a630_gmu.bin", + }, + .gmem = SZ_1M, + .init = a6xx_gpu_init, }, }; @@ -127,6 +137,8 @@ MODULE_FIRMWARE("qcom/a530_zap.mdt"); MODULE_FIRMWARE("qcom/a530_zap.b00"); MODULE_FIRMWARE("qcom/a530_zap.b01"); MODULE_FIRMWARE("qcom/a530_zap.b02"); +MODULE_FIRMWARE("qcom/a630_sqe.fw"); +MODULE_FIRMWARE("qcom/a630_gmu.bin"); static inline bool _rev_match(uint8_t entry, uint8_t id) { @@ -155,6 +167,7 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev) struct msm_drm_private *priv = dev->dev_private; struct platform_device *pdev = priv->gpu_pdev; struct msm_gpu *gpu = NULL; + struct adreno_gpu *adreno_gpu; int ret; if (pdev) @@ -165,7 +178,27 @@ struct msm_gpu *adreno_load_gpu(struct drm_device *dev) return NULL; } - pm_runtime_get_sync(&pdev->dev); + adreno_gpu = to_adreno_gpu(gpu); + + /* + * The number one reason for HW init to fail is if the firmware isn't + * loaded yet. Try that first and don't bother continuing on + * otherwise + */ + + ret = adreno_load_fw(adreno_gpu); + if (ret) + return NULL; + + /* Make sure pm runtime is active and reset any previous errors */ + pm_runtime_set_active(&pdev->dev); + + ret = pm_runtime_get_sync(&pdev->dev); + if (ret < 0) { + dev_err(dev->dev, "Couldn't power up the GPU: %d\n", ret); + return NULL; + } + mutex_lock(&dev->struct_mutex); ret = msm_gpu_hw_init(gpu); mutex_unlock(&dev->struct_mutex); diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.c b/drivers/gpu/drm/msm/adreno/adreno_gpu.c index 38ac50b73829..da1363a0c54d 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.c +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.c @@ -18,7 +18,9 @@ */ #include <linux/ascii85.h> +#include <linux/kernel.h> #include <linux/pm_opp.h> +#include <linux/slab.h> #include "adreno_gpu.h" #include "msm_gem.h" #include "msm_mmu.h" @@ -71,10 +73,12 @@ adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname) { struct drm_device *drm = adreno_gpu->base.dev; const struct firmware *fw = NULL; - char newname[strlen("qcom/") + strlen(fwname) + 1]; + char *newname; int ret; - sprintf(newname, "qcom/%s", fwname); + newname = kasprintf(GFP_KERNEL, "qcom/%s", fwname); + if (!newname) + return ERR_PTR(-ENOMEM); /* * Try first to load from qcom/$fwfile using a direct load (to avoid @@ -88,11 +92,12 @@ adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname) dev_info(drm->dev, "loaded %s from new location\n", newname); adreno_gpu->fwloc = FW_LOCATION_NEW; - return fw; + goto out; } else if (adreno_gpu->fwloc != FW_LOCATION_UNKNOWN) { dev_err(drm->dev, "failed to load %s: %d\n", newname, ret); - return ERR_PTR(ret); + fw = ERR_PTR(ret); + goto out; } } @@ -107,11 +112,12 @@ adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname) dev_info(drm->dev, "loaded %s from legacy location\n", newname); adreno_gpu->fwloc = FW_LOCATION_LEGACY; - return fw; + goto out; } else if (adreno_gpu->fwloc != FW_LOCATION_UNKNOWN) { dev_err(drm->dev, "failed to load %s: %d\n", fwname, ret); - return ERR_PTR(ret); + fw = ERR_PTR(ret); + goto out; } } @@ -127,19 +133,23 @@ adreno_request_fw(struct adreno_gpu *adreno_gpu, const char *fwname) dev_info(drm->dev, "loaded %s with helper\n", newname); adreno_gpu->fwloc = FW_LOCATION_HELPER; - return fw; + goto out; } else if (adreno_gpu->fwloc != FW_LOCATION_UNKNOWN) { dev_err(drm->dev, "failed to load %s: %d\n", newname, ret); - return ERR_PTR(ret); + fw = ERR_PTR(ret); + goto out; } } dev_err(drm->dev, "failed to load %s\n", fwname); - return ERR_PTR(-ENOENT); + fw = ERR_PTR(-ENOENT); +out: + kfree(newname); + return fw; } -static int adreno_load_fw(struct adreno_gpu *adreno_gpu) +int adreno_load_fw(struct adreno_gpu *adreno_gpu) { int i; diff --git a/drivers/gpu/drm/msm/adreno/adreno_gpu.h b/drivers/gpu/drm/msm/adreno/adreno_gpu.h index 4406776597fd..de6e6ee42fba 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_gpu.h +++ b/drivers/gpu/drm/msm/adreno/adreno_gpu.h @@ -50,7 +50,9 @@ enum adreno_regs { enum { ADRENO_FW_PM4 = 0, + ADRENO_FW_SQE = 0, /* a6xx */ ADRENO_FW_PFP = 1, + ADRENO_FW_GMU = 1, /* a6xx */ ADRENO_FW_GPMU = 2, ADRENO_FW_MAX, }; @@ -228,7 +230,7 @@ int adreno_gpu_init(struct drm_device *drm, struct platform_device *pdev, struct adreno_gpu *gpu, const struct adreno_gpu_funcs *funcs, int nr_rings); void adreno_gpu_cleanup(struct adreno_gpu *gpu); - +int adreno_load_fw(struct adreno_gpu *adreno_gpu); void adreno_gpu_state_destroy(struct msm_gpu_state *state); @@ -335,6 +337,7 @@ static inline void adreno_gpu_write(struct adreno_gpu *gpu, struct msm_gpu *a3xx_gpu_init(struct drm_device *dev); struct msm_gpu *a4xx_gpu_init(struct drm_device *dev); struct msm_gpu *a5xx_gpu_init(struct drm_device *dev); +struct msm_gpu *a6xx_gpu_init(struct drm_device *dev); static inline void adreno_gpu_write64(struct adreno_gpu *gpu, enum adreno_regs lo, enum adreno_regs hi, u64 data) diff --git a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h index fb605a3534cf..03a91e10b310 100644 --- a/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h +++ b/drivers/gpu/drm/msm/adreno/adreno_pm4.xml.h @@ -8,17 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/adreno.xml ( 431 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a2xx.xml ( 37162 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_common.xml ( 13324 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/adreno_pm4.xml ( 31866 bytes, from 2017-06-06 18:26:14) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a4xx.xml ( 111898 bytes, from 2017-06-06 18:23:59) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/a5xx.xml ( 139480 bytes, from 2017-06-16 12:44:39) -- /home/robclark/src/freedreno/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/adreno.xml ( 501 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a2xx.xml ( 36805 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_common.xml ( 13634 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/adreno_pm4.xml ( 42393 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a3xx.xml ( 83840 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a4xx.xml ( 112086 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/a5xx.xml ( 147240 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx.xml ( 101627 bytes, from 2018-08-06 18:45:45) +- /home/robclark/src/envytools/rnndb/adreno/a6xx_gmu.xml ( 10431 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/adreno/ocmem.xml ( 1773 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) @@ -71,7 +73,8 @@ enum vgt_event_type { FLUSH_SO_1 = 18, FLUSH_SO_2 = 19, FLUSH_SO_3 = 20, - UNK_19 = 25, + PC_CCU_INVALIDATE_DEPTH = 24, + PC_CCU_INVALIDATE_COLOR = 25, UNK_1C = 28, UNK_1D = 29, BLIT = 30, @@ -199,9 +202,12 @@ enum adreno_pm4_type3_packets { CP_WAIT_MEM_WRITES = 18, CP_COND_REG_EXEC = 71, CP_MEM_TO_REG = 66, + CP_EXEC_CS_INDIRECT = 65, CP_EXEC_CS = 51, CP_PERFCOUNTER_ACTION = 80, CP_SMMU_TABLE_UPDATE = 83, + CP_SET_MARKER = 101, + CP_SET_PSEUDO_REG = 86, CP_CONTEXT_REG_BUNCH = 92, CP_YIELD_ENABLE = 28, CP_SKIP_IB2_ENABLE_GLOBAL = 29, @@ -215,7 +221,10 @@ enum adreno_pm4_type3_packets { CP_COMPUTE_CHECKPOINT = 110, CP_MEM_TO_MEM = 115, CP_BLIT = 44, - CP_UNK_39 = 57, + CP_REG_TEST = 57, + CP_SET_MODE = 99, + CP_LOAD_STATE6_GEOM = 50, + CP_LOAD_STATE6_FRAG = 52, IN_IB_PREFETCH_END = 23, IN_SUBBLK_PREFETCH = 31, IN_INSTR_PREFETCH = 32, @@ -224,6 +233,11 @@ enum adreno_pm4_type3_packets { IN_INCR_UPDT_STATE = 85, IN_INCR_UPDT_CONST = 86, IN_INCR_UPDT_INSTR = 87, + PKT4 = 4, + CP_UNK_A6XX_14 = 20, + CP_UNK_A6XX_36 = 54, + CP_UNK_A6XX_55 = 85, + UNK_A6XX_6D = 109, }; enum adreno_state_block { @@ -278,6 +292,33 @@ enum a4xx_state_src { SS4_INDIRECT = 2, }; +enum a6xx_state_block { + SB6_VS_TEX = 0, + SB6_HS_TEX = 1, + SB6_DS_TEX = 2, + SB6_GS_TEX = 3, + SB6_FS_TEX = 4, + SB6_CS_TEX = 5, + SB6_VS_SHADER = 8, + SB6_HS_SHADER = 9, + SB6_DS_SHADER = 10, + SB6_GS_SHADER = 11, + SB6_FS_SHADER = 12, + SB6_CS_SHADER = 13, + SB6_SSBO = 14, + SB6_CS_SSBO = 15, +}; + +enum a6xx_state_type { + ST6_SHADER = 0, + ST6_CONSTANTS = 1, +}; + +enum a6xx_state_src { + SS6_DIRECT = 0, + SS6_INDIRECT = 2, +}; + enum a4xx_index_size { INDEX4_SIZE_8_BIT = 0, INDEX4_SIZE_16_BIT = 1, @@ -300,6 +341,7 @@ enum render_mode_cmd { GMEM = 3, BLIT2D = 5, BLIT2DSCALE = 7, + END2D = 8, }; enum cp_blit_cmd { @@ -308,6 +350,22 @@ enum cp_blit_cmd { BLIT_OP_SCALE = 3, }; +enum a6xx_render_mode { + RM6_BYPASS = 1, + RM6_BINNING = 2, + RM6_GMEM = 4, + RM6_BLIT2D = 5, + RM6_RESOLVE = 6, +}; + +enum pseudo_reg { + SMMU_INFO = 0, + NON_SECURE_SAVE_ADDR = 1, + SECURE_SAVE_ADDR = 2, + NON_PRIV_SAVE_ADDR = 3, + COUNTER = 4, +}; + #define REG_CP_LOAD_STATE_0 0x00000000 #define CP_LOAD_STATE_0_DST_OFF__MASK 0x0000ffff #define CP_LOAD_STATE_0_DST_OFF__SHIFT 0 @@ -349,7 +407,7 @@ static inline uint32_t CP_LOAD_STATE_1_EXT_SRC_ADDR(uint32_t val) } #define REG_CP_LOAD_STATE4_0 0x00000000 -#define CP_LOAD_STATE4_0_DST_OFF__MASK 0x0000ffff +#define CP_LOAD_STATE4_0_DST_OFF__MASK 0x00003fff #define CP_LOAD_STATE4_0_DST_OFF__SHIFT 0 static inline uint32_t CP_LOAD_STATE4_0_DST_OFF(uint32_t val) { @@ -396,6 +454,54 @@ static inline uint32_t CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI(uint32_t val) return ((val) << CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI__SHIFT) & CP_LOAD_STATE4_2_EXT_SRC_ADDR_HI__MASK; } +#define REG_CP_LOAD_STATE6_0 0x00000000 +#define CP_LOAD_STATE6_0_DST_OFF__MASK 0x00003fff +#define CP_LOAD_STATE6_0_DST_OFF__SHIFT 0 +static inline uint32_t CP_LOAD_STATE6_0_DST_OFF(uint32_t val) +{ + return ((val) << CP_LOAD_STATE6_0_DST_OFF__SHIFT) & CP_LOAD_STATE6_0_DST_OFF__MASK; +} +#define CP_LOAD_STATE6_0_STATE_TYPE__MASK 0x00004000 +#define CP_LOAD_STATE6_0_STATE_TYPE__SHIFT 14 +static inline uint32_t CP_LOAD_STATE6_0_STATE_TYPE(enum a6xx_state_type val) +{ + return ((val) << CP_LOAD_STATE6_0_STATE_TYPE__SHIFT) & CP_LOAD_STATE6_0_STATE_TYPE__MASK; +} +#define CP_LOAD_STATE6_0_STATE_SRC__MASK 0x00030000 +#define CP_LOAD_STATE6_0_STATE_SRC__SHIFT 16 +static inline uint32_t CP_LOAD_STATE6_0_STATE_SRC(enum a6xx_state_src val) +{ + return ((val) << CP_LOAD_STATE6_0_STATE_SRC__SHIFT) & CP_LOAD_STATE6_0_STATE_SRC__MASK; +} +#define CP_LOAD_STATE6_0_STATE_BLOCK__MASK 0x003c0000 +#define CP_LOAD_STATE6_0_STATE_BLOCK__SHIFT 18 +static inline uint32_t CP_LOAD_STATE6_0_STATE_BLOCK(enum a6xx_state_block val) +{ + return ((val) << CP_LOAD_STATE6_0_STATE_BLOCK__SHIFT) & CP_LOAD_STATE6_0_STATE_BLOCK__MASK; +} +#define CP_LOAD_STATE6_0_NUM_UNIT__MASK 0xffc00000 +#define CP_LOAD_STATE6_0_NUM_UNIT__SHIFT 22 +static inline uint32_t CP_LOAD_STATE6_0_NUM_UNIT(uint32_t val) +{ + return ((val) << CP_LOAD_STATE6_0_NUM_UNIT__SHIFT) & CP_LOAD_STATE6_0_NUM_UNIT__MASK; +} + +#define REG_CP_LOAD_STATE6_1 0x00000001 +#define CP_LOAD_STATE6_1_EXT_SRC_ADDR__MASK 0xfffffffc +#define CP_LOAD_STATE6_1_EXT_SRC_ADDR__SHIFT 2 +static inline uint32_t CP_LOAD_STATE6_1_EXT_SRC_ADDR(uint32_t val) +{ + return ((val >> 2) << CP_LOAD_STATE6_1_EXT_SRC_ADDR__SHIFT) & CP_LOAD_STATE6_1_EXT_SRC_ADDR__MASK; +} + +#define REG_CP_LOAD_STATE6_2 0x00000002 +#define CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__MASK 0xffffffff +#define CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__SHIFT 0 +static inline uint32_t CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI(uint32_t val) +{ + return ((val) << CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__SHIFT) & CP_LOAD_STATE6_2_EXT_SRC_ADDR_HI__MASK; +} + #define REG_CP_DRAW_INDX_0 0x00000000 #define CP_DRAW_INDX_0_VIZ_QUERY__MASK 0xffffffff #define CP_DRAW_INDX_0_VIZ_QUERY__SHIFT 0 @@ -580,6 +686,153 @@ static inline uint32_t CP_DRAW_INDX_OFFSET_5_INDX_SIZE(uint32_t val) return ((val) << CP_DRAW_INDX_OFFSET_5_INDX_SIZE__SHIFT) & CP_DRAW_INDX_OFFSET_5_INDX_SIZE__MASK; } +#define REG_A4XX_CP_DRAW_INDIRECT_0 0x00000000 +#define A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__MASK 0x0000003f +#define A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__SHIFT 0 +static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE(enum pc_di_primtype val) +{ + return ((val) << A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_PRIM_TYPE__MASK; +} +#define A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__MASK 0x000000c0 +#define A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__SHIFT 6 +static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT(enum pc_di_src_sel val) +{ + return ((val) << A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_SOURCE_SELECT__MASK; +} +#define A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__MASK 0x00000300 +#define A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__SHIFT 8 +static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_VIS_CULL(enum pc_di_vis_cull_mode val) +{ + return ((val) << A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_VIS_CULL__MASK; +} +#define A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__MASK 0x00000c00 +#define A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__SHIFT 10 +static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE(enum a4xx_index_size val) +{ + return ((val) << A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_INDEX_SIZE__MASK; +} +#define A4XX_CP_DRAW_INDIRECT_0_TESS_MODE__MASK 0x01f00000 +#define A4XX_CP_DRAW_INDIRECT_0_TESS_MODE__SHIFT 20 +static inline uint32_t A4XX_CP_DRAW_INDIRECT_0_TESS_MODE(uint32_t val) +{ + return ((val) << A4XX_CP_DRAW_INDIRECT_0_TESS_MODE__SHIFT) & A4XX_CP_DRAW_INDIRECT_0_TESS_MODE__MASK; +} + +#define REG_A4XX_CP_DRAW_INDIRECT_1 0x00000001 +#define A4XX_CP_DRAW_INDIRECT_1_INDIRECT__MASK 0xffffffff +#define A4XX_CP_DRAW_INDIRECT_1_INDIRECT__SHIFT 0 +static inline uint32_t A4XX_CP_DRAW_INDIRECT_1_INDIRECT(uint32_t val) +{ + return ((val) << A4XX_CP_DRAW_INDIRECT_1_INDIRECT__SHIFT) & A4XX_CP_DRAW_INDIRECT_1_INDIRECT__MASK; +} + + +#define REG_A5XX_CP_DRAW_INDIRECT_2 0x00000002 +#define A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__MASK 0xffffffff +#define A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__SHIFT 0 +static inline uint32_t A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI(uint32_t val) +{ + return ((val) << A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__SHIFT) & A5XX_CP_DRAW_INDIRECT_2_INDIRECT_HI__MASK; +} + +#define REG_A4XX_CP_DRAW_INDX_INDIRECT_0 0x00000000 +#define A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__MASK 0x0000003f +#define A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__SHIFT 0 +static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE(enum pc_di_primtype val) +{ + return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_PRIM_TYPE__MASK; +} +#define A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__MASK 0x000000c0 +#define A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__SHIFT 6 +static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT(enum pc_di_src_sel val) +{ + return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_SOURCE_SELECT__MASK; +} +#define A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__MASK 0x00000300 +#define A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__SHIFT 8 +static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL(enum pc_di_vis_cull_mode val) +{ + return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_VIS_CULL__MASK; +} +#define A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__MASK 0x00000c00 +#define A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__SHIFT 10 +static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE(enum a4xx_index_size val) +{ + return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_INDEX_SIZE__MASK; +} +#define A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE__MASK 0x01f00000 +#define A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE__SHIFT 20 +static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE(uint32_t val) +{ + return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_0_TESS_MODE__MASK; +} + + +#define REG_A4XX_CP_DRAW_INDX_INDIRECT_1 0x00000001 +#define A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__MASK 0xffffffff +#define A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__SHIFT 0 +static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE(uint32_t val) +{ + return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE__MASK; +} + +#define REG_A4XX_CP_DRAW_INDX_INDIRECT_2 0x00000002 +#define A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__MASK 0xffffffff +#define A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__SHIFT 0 +static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE(uint32_t val) +{ + return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_2_INDX_SIZE__MASK; +} + +#define REG_A4XX_CP_DRAW_INDX_INDIRECT_3 0x00000003 +#define A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__MASK 0xffffffff +#define A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__SHIFT 0 +static inline uint32_t A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT(uint32_t val) +{ + return ((val) << A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__SHIFT) & A4XX_CP_DRAW_INDX_INDIRECT_3_INDIRECT__MASK; +} + + +#define REG_A5XX_CP_DRAW_INDX_INDIRECT_1 0x00000001 +#define A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__MASK 0xffffffff +#define A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__SHIFT 0 +static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO(uint32_t val) +{ + return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_1_INDX_BASE_LO__MASK; +} + +#define REG_A5XX_CP_DRAW_INDX_INDIRECT_2 0x00000002 +#define A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__MASK 0xffffffff +#define A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__SHIFT 0 +static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI(uint32_t val) +{ + return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_2_INDX_BASE_HI__MASK; +} + +#define REG_A5XX_CP_DRAW_INDX_INDIRECT_3 0x00000003 +#define A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__MASK 0xffffffff +#define A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__SHIFT 0 +static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES(uint32_t val) +{ + return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_3_MAX_INDICES__MASK; +} + +#define REG_A5XX_CP_DRAW_INDX_INDIRECT_4 0x00000004 +#define A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__MASK 0xffffffff +#define A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__SHIFT 0 +static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO(uint32_t val) +{ + return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_4_INDIRECT_LO__MASK; +} + +#define REG_A5XX_CP_DRAW_INDX_INDIRECT_5 0x00000005 +#define A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__MASK 0xffffffff +#define A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__SHIFT 0 +static inline uint32_t A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI(uint32_t val) +{ + return ((val) << A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__SHIFT) & A5XX_CP_DRAW_INDX_INDIRECT_5_INDIRECT_HI__MASK; +} + static inline uint32_t REG_CP_SET_DRAW_STATE_(uint32_t i0) { return 0x00000000 + 0x3*i0; } static inline uint32_t REG_CP_SET_DRAW_STATE__0(uint32_t i0) { return 0x00000000 + 0x3*i0; } @@ -593,6 +846,12 @@ static inline uint32_t CP_SET_DRAW_STATE__0_COUNT(uint32_t val) #define CP_SET_DRAW_STATE__0_DISABLE 0x00020000 #define CP_SET_DRAW_STATE__0_DISABLE_ALL_GROUPS 0x00040000 #define CP_SET_DRAW_STATE__0_LOAD_IMMED 0x00080000 +#define CP_SET_DRAW_STATE__0_ENABLE_MASK__MASK 0x00f00000 +#define CP_SET_DRAW_STATE__0_ENABLE_MASK__SHIFT 20 +static inline uint32_t CP_SET_DRAW_STATE__0_ENABLE_MASK(uint32_t val) +{ + return ((val) << CP_SET_DRAW_STATE__0_ENABLE_MASK__SHIFT) & CP_SET_DRAW_STATE__0_ENABLE_MASK__MASK; +} #define CP_SET_DRAW_STATE__0_GROUP_ID__MASK 0x1f000000 #define CP_SET_DRAW_STATE__0_GROUP_ID__SHIFT 24 static inline uint32_t CP_SET_DRAW_STATE__0_GROUP_ID(uint32_t val) @@ -708,6 +967,22 @@ static inline uint32_t CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI(uint32_t val) return ((val) << CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI__SHIFT) & CP_SET_BIN_DATA5_4_BIN_SIZE_ADDRESS_HI__MASK; } +#define REG_CP_SET_BIN_DATA5_5 0x00000005 +#define CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO__MASK 0xffffffff +#define CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO__SHIFT 0 +static inline uint32_t CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO(uint32_t val) +{ + return ((val) << CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO__SHIFT) & CP_SET_BIN_DATA5_5_XXX_ADDRESS_LO__MASK; +} + +#define REG_CP_SET_BIN_DATA5_6 0x00000006 +#define CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI__MASK 0xffffffff +#define CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI__SHIFT 0 +static inline uint32_t CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI(uint32_t val) +{ + return ((val) << CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI__SHIFT) & CP_SET_BIN_DATA5_6_XXX_ADDRESS_HI__MASK; +} + #define REG_CP_REG_TO_MEM_0 0x00000000 #define CP_REG_TO_MEM_0_REG__MASK 0x0000ffff #define CP_REG_TO_MEM_0_REG__SHIFT 0 @@ -732,6 +1007,46 @@ static inline uint32_t CP_REG_TO_MEM_1_DEST(uint32_t val) return ((val) << CP_REG_TO_MEM_1_DEST__SHIFT) & CP_REG_TO_MEM_1_DEST__MASK; } +#define REG_CP_REG_TO_MEM_2 0x00000002 +#define CP_REG_TO_MEM_2_DEST_HI__MASK 0xffffffff +#define CP_REG_TO_MEM_2_DEST_HI__SHIFT 0 +static inline uint32_t CP_REG_TO_MEM_2_DEST_HI(uint32_t val) +{ + return ((val) << CP_REG_TO_MEM_2_DEST_HI__SHIFT) & CP_REG_TO_MEM_2_DEST_HI__MASK; +} + +#define REG_CP_MEM_TO_REG_0 0x00000000 +#define CP_MEM_TO_REG_0_REG__MASK 0x0000ffff +#define CP_MEM_TO_REG_0_REG__SHIFT 0 +static inline uint32_t CP_MEM_TO_REG_0_REG(uint32_t val) +{ + return ((val) << CP_MEM_TO_REG_0_REG__SHIFT) & CP_MEM_TO_REG_0_REG__MASK; +} +#define CP_MEM_TO_REG_0_CNT__MASK 0x3ff80000 +#define CP_MEM_TO_REG_0_CNT__SHIFT 19 +static inline uint32_t CP_MEM_TO_REG_0_CNT(uint32_t val) +{ + return ((val) << CP_MEM_TO_REG_0_CNT__SHIFT) & CP_MEM_TO_REG_0_CNT__MASK; +} +#define CP_MEM_TO_REG_0_64B 0x40000000 +#define CP_MEM_TO_REG_0_ACCUMULATE 0x80000000 + +#define REG_CP_MEM_TO_REG_1 0x00000001 +#define CP_MEM_TO_REG_1_SRC__MASK 0xffffffff +#define CP_MEM_TO_REG_1_SRC__SHIFT 0 +static inline uint32_t CP_MEM_TO_REG_1_SRC(uint32_t val) +{ + return ((val) << CP_MEM_TO_REG_1_SRC__SHIFT) & CP_MEM_TO_REG_1_SRC__MASK; +} + +#define REG_CP_MEM_TO_REG_2 0x00000002 +#define CP_MEM_TO_REG_2_SRC_HI__MASK 0xffffffff +#define CP_MEM_TO_REG_2_SRC_HI__SHIFT 0 +static inline uint32_t CP_MEM_TO_REG_2_SRC_HI(uint32_t val) +{ + return ((val) << CP_MEM_TO_REG_2_SRC_HI__SHIFT) & CP_MEM_TO_REG_2_SRC_HI__MASK; +} + #define REG_CP_MEM_TO_MEM_0 0x00000000 #define CP_MEM_TO_MEM_0_NEG_A 0x00000001 #define CP_MEM_TO_MEM_0_NEG_B 0x00000002 @@ -953,15 +1268,15 @@ static inline uint32_t CP_COMPUTE_CHECKPOINT_1_ADDR_0_HI(uint32_t val) #define REG_CP_COMPUTE_CHECKPOINT_2 0x00000002 #define REG_CP_COMPUTE_CHECKPOINT_3 0x00000003 - -#define REG_CP_COMPUTE_CHECKPOINT_4 0x00000004 -#define CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__MASK 0xffffffff -#define CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__SHIFT 0 -static inline uint32_t CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN(uint32_t val) +#define CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN__MASK 0xffffffff +#define CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN__SHIFT 0 +static inline uint32_t CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN(uint32_t val) { - return ((val) << CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__SHIFT) & CP_COMPUTE_CHECKPOINT_4_ADDR_1_LEN__MASK; + return ((val) << CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN__SHIFT) & CP_COMPUTE_CHECKPOINT_3_ADDR_1_LEN__MASK; } +#define REG_CP_COMPUTE_CHECKPOINT_4 0x00000004 + #define REG_CP_COMPUTE_CHECKPOINT_5 0x00000005 #define CP_COMPUTE_CHECKPOINT_5_ADDR_1_LO__MASK 0xffffffff #define CP_COMPUTE_CHECKPOINT_5_ADDR_1_LO__SHIFT 0 @@ -978,6 +1293,8 @@ static inline uint32_t CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI(uint32_t val) return ((val) << CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI__SHIFT) & CP_COMPUTE_CHECKPOINT_6_ADDR_1_HI__MASK; } +#define REG_CP_COMPUTE_CHECKPOINT_7 0x00000007 + #define REG_CP_PERFCOUNTER_ACTION_0 0x00000000 #define REG_CP_PERFCOUNTER_ACTION_1 0x00000001 @@ -1032,13 +1349,13 @@ static inline uint32_t CP_BLIT_0_OP(enum cp_blit_cmd val) } #define REG_CP_BLIT_1 0x00000001 -#define CP_BLIT_1_SRC_X1__MASK 0x0000ffff +#define CP_BLIT_1_SRC_X1__MASK 0x00003fff #define CP_BLIT_1_SRC_X1__SHIFT 0 static inline uint32_t CP_BLIT_1_SRC_X1(uint32_t val) { return ((val) << CP_BLIT_1_SRC_X1__SHIFT) & CP_BLIT_1_SRC_X1__MASK; } -#define CP_BLIT_1_SRC_Y1__MASK 0xffff0000 +#define CP_BLIT_1_SRC_Y1__MASK 0x3fff0000 #define CP_BLIT_1_SRC_Y1__SHIFT 16 static inline uint32_t CP_BLIT_1_SRC_Y1(uint32_t val) { @@ -1046,13 +1363,13 @@ static inline uint32_t CP_BLIT_1_SRC_Y1(uint32_t val) } #define REG_CP_BLIT_2 0x00000002 -#define CP_BLIT_2_SRC_X2__MASK 0x0000ffff +#define CP_BLIT_2_SRC_X2__MASK 0x00003fff #define CP_BLIT_2_SRC_X2__SHIFT 0 static inline uint32_t CP_BLIT_2_SRC_X2(uint32_t val) { return ((val) << CP_BLIT_2_SRC_X2__SHIFT) & CP_BLIT_2_SRC_X2__MASK; } -#define CP_BLIT_2_SRC_Y2__MASK 0xffff0000 +#define CP_BLIT_2_SRC_Y2__MASK 0x3fff0000 #define CP_BLIT_2_SRC_Y2__SHIFT 16 static inline uint32_t CP_BLIT_2_SRC_Y2(uint32_t val) { @@ -1060,13 +1377,13 @@ static inline uint32_t CP_BLIT_2_SRC_Y2(uint32_t val) } #define REG_CP_BLIT_3 0x00000003 -#define CP_BLIT_3_DST_X1__MASK 0x0000ffff +#define CP_BLIT_3_DST_X1__MASK 0x00003fff #define CP_BLIT_3_DST_X1__SHIFT 0 static inline uint32_t CP_BLIT_3_DST_X1(uint32_t val) { return ((val) << CP_BLIT_3_DST_X1__SHIFT) & CP_BLIT_3_DST_X1__MASK; } -#define CP_BLIT_3_DST_Y1__MASK 0xffff0000 +#define CP_BLIT_3_DST_Y1__MASK 0x3fff0000 #define CP_BLIT_3_DST_Y1__SHIFT 16 static inline uint32_t CP_BLIT_3_DST_Y1(uint32_t val) { @@ -1074,13 +1391,13 @@ static inline uint32_t CP_BLIT_3_DST_Y1(uint32_t val) } #define REG_CP_BLIT_4 0x00000004 -#define CP_BLIT_4_DST_X2__MASK 0x0000ffff +#define CP_BLIT_4_DST_X2__MASK 0x00003fff #define CP_BLIT_4_DST_X2__SHIFT 0 static inline uint32_t CP_BLIT_4_DST_X2(uint32_t val) { return ((val) << CP_BLIT_4_DST_X2__SHIFT) & CP_BLIT_4_DST_X2__MASK; } -#define CP_BLIT_4_DST_Y2__MASK 0xffff0000 +#define CP_BLIT_4_DST_Y2__MASK 0x3fff0000 #define CP_BLIT_4_DST_Y2__SHIFT 16 static inline uint32_t CP_BLIT_4_DST_Y2(uint32_t val) { @@ -1113,5 +1430,129 @@ static inline uint32_t CP_EXEC_CS_3_NGROUPS_Z(uint32_t val) return ((val) << CP_EXEC_CS_3_NGROUPS_Z__SHIFT) & CP_EXEC_CS_3_NGROUPS_Z__MASK; } +#define REG_A4XX_CP_EXEC_CS_INDIRECT_0 0x00000000 + + +#define REG_A4XX_CP_EXEC_CS_INDIRECT_1 0x00000001 +#define A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__MASK 0xffffffff +#define A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__SHIFT 0 +static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_1_ADDR(uint32_t val) +{ + return ((val) << A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_1_ADDR__MASK; +} + +#define REG_A4XX_CP_EXEC_CS_INDIRECT_2 0x00000002 +#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__MASK 0x00000ffc +#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__SHIFT 2 +static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX(uint32_t val) +{ + return ((val) << A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEX__MASK; +} +#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__MASK 0x003ff000 +#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__SHIFT 12 +static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY(uint32_t val) +{ + return ((val) << A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEY__MASK; +} +#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__MASK 0xffc00000 +#define A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__SHIFT 22 +static inline uint32_t A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ(uint32_t val) +{ + return ((val) << A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__SHIFT) & A4XX_CP_EXEC_CS_INDIRECT_2_LOCALSIZEZ__MASK; +} + + +#define REG_A5XX_CP_EXEC_CS_INDIRECT_1 0x00000001 +#define A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__MASK 0xffffffff +#define A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__SHIFT 0 +static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO(uint32_t val) +{ + return ((val) << A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_1_ADDR_LO__MASK; +} + +#define REG_A5XX_CP_EXEC_CS_INDIRECT_2 0x00000002 +#define A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__MASK 0xffffffff +#define A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__SHIFT 0 +static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI(uint32_t val) +{ + return ((val) << A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_2_ADDR_HI__MASK; +} + +#define REG_A5XX_CP_EXEC_CS_INDIRECT_3 0x00000003 +#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__MASK 0x00000ffc +#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__SHIFT 2 +static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX(uint32_t val) +{ + return ((val) << A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEX__MASK; +} +#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__MASK 0x003ff000 +#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__SHIFT 12 +static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY(uint32_t val) +{ + return ((val) << A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEY__MASK; +} +#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__MASK 0xffc00000 +#define A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__SHIFT 22 +static inline uint32_t A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ(uint32_t val) +{ + return ((val) << A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__SHIFT) & A5XX_CP_EXEC_CS_INDIRECT_3_LOCALSIZEZ__MASK; +} + +#define REG_A2XX_CP_SET_MARKER_0 0x00000000 +#define A2XX_CP_SET_MARKER_0_MARKER__MASK 0x0000000f +#define A2XX_CP_SET_MARKER_0_MARKER__SHIFT 0 +static inline uint32_t A2XX_CP_SET_MARKER_0_MARKER(uint32_t val) +{ + return ((val) << A2XX_CP_SET_MARKER_0_MARKER__SHIFT) & A2XX_CP_SET_MARKER_0_MARKER__MASK; +} +#define A2XX_CP_SET_MARKER_0_MODE__MASK 0x0000000f +#define A2XX_CP_SET_MARKER_0_MODE__SHIFT 0 +static inline uint32_t A2XX_CP_SET_MARKER_0_MODE(enum a6xx_render_mode val) +{ + return ((val) << A2XX_CP_SET_MARKER_0_MODE__SHIFT) & A2XX_CP_SET_MARKER_0_MODE__MASK; +} +#define A2XX_CP_SET_MARKER_0_IFPC 0x00000100 + +static inline uint32_t REG_A2XX_CP_SET_PSEUDO_REG_(uint32_t i0) { return 0x00000000 + 0x3*i0; } + +static inline uint32_t REG_A2XX_CP_SET_PSEUDO_REG__0(uint32_t i0) { return 0x00000000 + 0x3*i0; } +#define A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__MASK 0x00000007 +#define A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__SHIFT 0 +static inline uint32_t A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG(enum pseudo_reg val) +{ + return ((val) << A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__SHIFT) & A2XX_CP_SET_PSEUDO_REG__0_PSEUDO_REG__MASK; +} + +static inline uint32_t REG_A2XX_CP_SET_PSEUDO_REG__1(uint32_t i0) { return 0x00000001 + 0x3*i0; } +#define A2XX_CP_SET_PSEUDO_REG__1_LO__MASK 0xffffffff +#define A2XX_CP_SET_PSEUDO_REG__1_LO__SHIFT 0 +static inline uint32_t A2XX_CP_SET_PSEUDO_REG__1_LO(uint32_t val) +{ + return ((val) << A2XX_CP_SET_PSEUDO_REG__1_LO__SHIFT) & A2XX_CP_SET_PSEUDO_REG__1_LO__MASK; +} + +static inline uint32_t REG_A2XX_CP_SET_PSEUDO_REG__2(uint32_t i0) { return 0x00000002 + 0x3*i0; } +#define A2XX_CP_SET_PSEUDO_REG__2_HI__MASK 0xffffffff +#define A2XX_CP_SET_PSEUDO_REG__2_HI__SHIFT 0 +static inline uint32_t A2XX_CP_SET_PSEUDO_REG__2_HI(uint32_t val) +{ + return ((val) << A2XX_CP_SET_PSEUDO_REG__2_HI__SHIFT) & A2XX_CP_SET_PSEUDO_REG__2_HI__MASK; +} + +#define REG_A2XX_CP_REG_TEST_0 0x00000000 +#define A2XX_CP_REG_TEST_0_REG__MASK 0x00000fff +#define A2XX_CP_REG_TEST_0_REG__SHIFT 0 +static inline uint32_t A2XX_CP_REG_TEST_0_REG(uint32_t val) +{ + return ((val) << A2XX_CP_REG_TEST_0_REG__SHIFT) & A2XX_CP_REG_TEST_0_REG__MASK; +} +#define A2XX_CP_REG_TEST_0_BIT__MASK 0x01f00000 +#define A2XX_CP_REG_TEST_0_BIT__SHIFT 20 +static inline uint32_t A2XX_CP_REG_TEST_0_BIT(uint32_t val) +{ + return ((val) << A2XX_CP_REG_TEST_0_BIT__SHIFT) & A2XX_CP_REG_TEST_0_BIT__MASK; +} +#define A2XX_CP_REG_TEST_0_UNK25 0x02000000 + #endif /* ADRENO_PM4_XML */ diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c index 0bd3eda93e22..1b4de3486ef9 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_encoder.c @@ -421,7 +421,7 @@ int dpu_encoder_helper_unregister_irq(struct dpu_encoder_phys *phys_enc, ret = dpu_core_irq_disable(phys_enc->dpu_kms, &irq->irq_idx, 1); if (ret) { - DRM_ERROR("diable failed id=%u, intr=%d, hw=%d, irq=%d ret=%d", + DRM_ERROR("disable failed id=%u, intr=%d, hw=%d, irq=%d ret=%d", DRMID(phys_enc->parent), intr_idx, irq->hw_idx, irq->irq_idx, ret); } @@ -2444,6 +2444,8 @@ int dpu_encoder_wait_for_event(struct drm_encoder *drm_enc, for (i = 0; i < dpu_enc->num_phys_encs; i++) { struct dpu_encoder_phys *phys = dpu_enc->phys_encs[i]; + if (!phys) + continue; switch (event) { case MSM_ENC_COMMIT_DONE: @@ -2461,7 +2463,7 @@ int dpu_encoder_wait_for_event(struct drm_encoder *drm_enc, return -EINVAL; }; - if (phys && fn_wait) { + if (fn_wait) { DPU_ATRACE_BEGIN("wait_for_completion_event"); ret = fn_wait(phys); DPU_ATRACE_END("wait_for_completion_event"); diff --git a/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.c b/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.c index a68f1249388c..a75eebca2f37 100644 --- a/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.c +++ b/drivers/gpu/drm/msm/disp/dpu1/dpu_power_handle.c @@ -121,7 +121,7 @@ void dpu_power_resource_deinit(struct platform_device *pdev, mutex_lock(&phandle->phandle_lock); list_for_each_entry_safe(curr_client, next_client, &phandle->power_client_clist, list) { - pr_err("cliend:%s-%d still registered with refcount:%d\n", + pr_err("client:%s-%d still registered with refcount:%d\n", curr_client->name, curr_client->id, curr_client->refcount); curr_client->active = false; diff --git a/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h b/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h index 576cea30d391..4b36b8954bae 100644 --- a/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h +++ b/drivers/gpu/drm/msm/disp/mdp4/mdp4.xml.h @@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 33004 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2017-06-16 12:32:42) -- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/msm.xml ( 676 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) diff --git a/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h b/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h index d9c10e02ee41..784d98989e3a 100644 --- a/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h +++ b/drivers/gpu/drm/msm/disp/mdp5/mdp5.xml.h @@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 33004 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2017-06-16 12:32:42) -- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/msm.xml ( 676 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) diff --git a/drivers/gpu/drm/msm/disp/mdp_common.xml.h b/drivers/gpu/drm/msm/disp/mdp_common.xml.h index 1494c407be44..d420c8044e23 100644 --- a/drivers/gpu/drm/msm/disp/mdp_common.xml.h +++ b/drivers/gpu/drm/msm/disp/mdp_common.xml.h @@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 33004 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2017-06-16 12:32:42) -- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/msm.xml ( 676 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) diff --git a/drivers/gpu/drm/msm/dsi/dsi.xml.h b/drivers/gpu/drm/msm/dsi/dsi.xml.h index f6a9471b70c8..21f489a737d7 100644 --- a/drivers/gpu/drm/msm/dsi/dsi.xml.h +++ b/drivers/gpu/drm/msm/dsi/dsi.xml.h @@ -8,8 +8,17 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /local/mnt/workspace/source_trees/envytools/rnndb/../rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-01-12 09:09:22) -- /local/mnt/workspace/source_trees/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2016-05-09 06:32:54) +- /home/robclark/src/envytools/rnndb/msm.xml ( 676 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2018-07-03 19:37:13) Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) diff --git a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h index 57cf7fa7f1c4..874265314413 100644 --- a/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h +++ b/drivers/gpu/drm/msm/dsi/mmss_cc.xml.h @@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 33004 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2017-06-16 12:32:42) -- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/msm.xml ( 676 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) diff --git a/drivers/gpu/drm/msm/dsi/sfpb.xml.h b/drivers/gpu/drm/msm/dsi/sfpb.xml.h index 9d4d1feaefd7..07c48ddb5301 100644 --- a/drivers/gpu/drm/msm/dsi/sfpb.xml.h +++ b/drivers/gpu/drm/msm/dsi/sfpb.xml.h @@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 33004 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2017-06-16 12:32:42) -- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/msm.xml ( 676 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) diff --git a/drivers/gpu/drm/msm/edp/edp.xml.h b/drivers/gpu/drm/msm/edp/edp.xml.h index f150d4a47707..9cb6e6fe9810 100644 --- a/drivers/gpu/drm/msm/edp/edp.xml.h +++ b/drivers/gpu/drm/msm/edp/edp.xml.h @@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 33004 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2017-06-16 12:32:42) -- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/msm.xml ( 676 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) diff --git a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h index ecebf8b623ab..3eff3ea3b271 100644 --- a/drivers/gpu/drm/msm/hdmi/hdmi.xml.h +++ b/drivers/gpu/drm/msm/hdmi/hdmi.xml.h @@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 33004 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2017-06-16 12:32:42) -- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/msm.xml ( 676 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) diff --git a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h index da646deedf4b..7717d4269662 100644 --- a/drivers/gpu/drm/msm/hdmi/qfprom.xml.h +++ b/drivers/gpu/drm/msm/hdmi/qfprom.xml.h @@ -8,19 +8,19 @@ http://github.com/freedreno/envytools/ git clone https://github.com/freedreno/envytools.git The rules-ng-ng source files this header was generated from are: -- /home/robclark/src/freedreno/envytools/rnndb/msm.xml ( 676 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/dsi.xml ( 33004 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2017-05-17 13:21:27) -- /home/robclark/src/freedreno/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2017-06-16 12:32:42) -- /home/robclark/src/freedreno/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2017-05-17 13:21:27) - -Copyright (C) 2013-2017 by the following authors: +- /home/robclark/src/envytools/rnndb/msm.xml ( 676 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/freedreno_copyright.xml ( 1572 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp4.xml ( 20915 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp_common.xml ( 2849 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/mdp/mdp5.xml ( 37411 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/dsi.xml ( 37239 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/sfpb.xml ( 602 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/dsi/mmss_cc.xml ( 1686 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/qfprom.xml ( 600 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/hdmi/hdmi.xml ( 41799 bytes, from 2018-07-03 19:37:13) +- /home/robclark/src/envytools/rnndb/edp/edp.xml ( 10416 bytes, from 2018-07-03 19:37:13) + +Copyright (C) 2013-2018 by the following authors: - Rob Clark <robdclark@gmail.com> (robclark) - Ilia Mirkin <imirkin@alum.mit.edu> (imirkin) diff --git a/drivers/gpu/drm/msm/msm_drv.c b/drivers/gpu/drm/msm/msm_drv.c index 46876bc8b707..c1abad8a8612 100644 --- a/drivers/gpu/drm/msm/msm_drv.c +++ b/drivers/gpu/drm/msm/msm_drv.c @@ -81,6 +81,63 @@ module_param(modeset, bool, 0600); * Util/helpers: */ +int msm_clk_bulk_get(struct device *dev, struct clk_bulk_data **bulk) +{ + struct property *prop; + const char *name; + struct clk_bulk_data *local; + int i = 0, ret, count; + + count = of_property_count_strings(dev->of_node, "clock-names"); + if (count < 1) + return 0; + + local = devm_kcalloc(dev, sizeof(struct clk_bulk_data *), + count, GFP_KERNEL); + if (!local) + return -ENOMEM; + + of_property_for_each_string(dev->of_node, "clock-names", prop, name) { + local[i].id = devm_kstrdup(dev, name, GFP_KERNEL); + if (!local[i].id) { + devm_kfree(dev, local); + return -ENOMEM; + } + + i++; + } + + ret = devm_clk_bulk_get(dev, count, local); + + if (ret) { + for (i = 0; i < count; i++) + devm_kfree(dev, (void *) local[i].id); + devm_kfree(dev, local); + + return ret; + } + + *bulk = local; + return count; +} + +struct clk *msm_clk_bulk_get_clock(struct clk_bulk_data *bulk, int count, + const char *name) +{ + int i; + char n[32]; + + snprintf(n, sizeof(n), "%s_clk", name); + + for (i = 0; bulk && i < count; i++) { + if (!strcmp(bulk[i].id, name) || !strcmp(bulk[i].id, n)) + return bulk[i].clk; + } + + + return NULL; +} + struct clk *msm_clk_get(struct platform_device *pdev, const char *name) { struct clk *clk; diff --git a/drivers/gpu/drm/msm/msm_drv.h b/drivers/gpu/drm/msm/msm_drv.h index 0cba86ed3f54..8e510d5c758a 100644 --- a/drivers/gpu/drm/msm/msm_drv.h +++ b/drivers/gpu/drm/msm/msm_drv.h @@ -263,7 +263,7 @@ void msm_gem_shrinker_cleanup(struct drm_device *dev); int msm_gem_mmap_obj(struct drm_gem_object *obj, struct vm_area_struct *vma); int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma); -int msm_gem_fault(struct vm_fault *vmf); +vm_fault_t msm_gem_fault(struct vm_fault *vmf); uint64_t msm_gem_mmap_offset(struct drm_gem_object *obj); int msm_gem_get_iova(struct drm_gem_object *obj, struct msm_gem_address_space *aspace, uint64_t *iova); @@ -387,6 +387,10 @@ static inline void msm_perf_debugfs_cleanup(struct msm_drm_private *priv) {} #endif struct clk *msm_clk_get(struct platform_device *pdev, const char *name); +int msm_clk_bulk_get(struct device *dev, struct clk_bulk_data **bulk); + +struct clk *msm_clk_bulk_get_clock(struct clk_bulk_data *bulk, int count, + const char *name); void __iomem *msm_ioremap(struct platform_device *pdev, const char *name, const char *dbgname); void msm_writel(u32 data, void __iomem *addr); diff --git a/drivers/gpu/drm/msm/msm_gem.c b/drivers/gpu/drm/msm/msm_gem.c index f583bb4222f9..f59ca27a4a35 100644 --- a/drivers/gpu/drm/msm/msm_gem.c +++ b/drivers/gpu/drm/msm/msm_gem.c @@ -219,7 +219,7 @@ int msm_gem_mmap(struct file *filp, struct vm_area_struct *vma) return msm_gem_mmap_obj(vma->vm_private_data, vma); } -int msm_gem_fault(struct vm_fault *vmf) +vm_fault_t msm_gem_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct drm_gem_object *obj = vma->vm_private_data; @@ -227,15 +227,18 @@ int msm_gem_fault(struct vm_fault *vmf) struct page **pages; unsigned long pfn; pgoff_t pgoff; - int ret; + int err; + vm_fault_t ret; /* * vm_ops.open/drm_gem_mmap_obj and close get and put * a reference on obj. So, we dont need to hold one here. */ - ret = mutex_lock_interruptible(&msm_obj->lock); - if (ret) + err = mutex_lock_interruptible(&msm_obj->lock); + if (err) { + ret = VM_FAULT_NOPAGE; goto out; + } if (WARN_ON(msm_obj->madv != MSM_MADV_WILLNEED)) { mutex_unlock(&msm_obj->lock); @@ -245,7 +248,7 @@ int msm_gem_fault(struct vm_fault *vmf) /* make sure we have pages attached now */ pages = get_pages(obj); if (IS_ERR(pages)) { - ret = PTR_ERR(pages); + ret = vmf_error(PTR_ERR(pages)); goto out_unlock; } @@ -257,27 +260,11 @@ int msm_gem_fault(struct vm_fault *vmf) VERB("Inserting %p pfn %lx, pa %lx", (void *)vmf->address, pfn, pfn << PAGE_SHIFT); - ret = vm_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV)); - + ret = vmf_insert_mixed(vma, vmf->address, __pfn_to_pfn_t(pfn, PFN_DEV)); out_unlock: mutex_unlock(&msm_obj->lock); out: - switch (ret) { - case -EAGAIN: - case 0: - case -ERESTARTSYS: - case -EINTR: - case -EBUSY: - /* - * EBUSY is ok: this just means that another thread - * already did the job. - */ - return VM_FAULT_NOPAGE; - case -ENOMEM: - return VM_FAULT_OOM; - default: - return VM_FAULT_SIGBUS; - } + return ret; } /** get mmap offset */ diff --git a/drivers/gpu/drm/msm/msm_gpu.c b/drivers/gpu/drm/msm/msm_gpu.c index f388944c93e2..5e808cfec345 100644 --- a/drivers/gpu/drm/msm/msm_gpu.c +++ b/drivers/gpu/drm/msm/msm_gpu.c @@ -88,7 +88,7 @@ static struct devfreq_dev_profile msm_devfreq_profile = { static void msm_devfreq_init(struct msm_gpu *gpu) { /* We need target support to do devfreq */ - if (!gpu->funcs->gpu_busy) + if (!gpu->funcs->gpu_busy || !gpu->core_clk) return; msm_devfreq_profile.initial_freq = gpu->fast_rate; @@ -142,8 +142,6 @@ static int disable_pwrrail(struct msm_gpu *gpu) static int enable_clk(struct msm_gpu *gpu) { - int i; - if (gpu->core_clk && gpu->fast_rate) clk_set_rate(gpu->core_clk, gpu->fast_rate); @@ -151,28 +149,12 @@ static int enable_clk(struct msm_gpu *gpu) if (gpu->rbbmtimer_clk) clk_set_rate(gpu->rbbmtimer_clk, 19200000); - for (i = gpu->nr_clocks - 1; i >= 0; i--) - if (gpu->grp_clks[i]) - clk_prepare(gpu->grp_clks[i]); - - for (i = gpu->nr_clocks - 1; i >= 0; i--) - if (gpu->grp_clks[i]) - clk_enable(gpu->grp_clks[i]); - - return 0; + return clk_bulk_prepare_enable(gpu->nr_clocks, gpu->grp_clks); } static int disable_clk(struct msm_gpu *gpu) { - int i; - - for (i = gpu->nr_clocks - 1; i >= 0; i--) - if (gpu->grp_clks[i]) - clk_disable(gpu->grp_clks[i]); - - for (i = gpu->nr_clocks - 1; i >= 0; i--) - if (gpu->grp_clks[i]) - clk_unprepare(gpu->grp_clks[i]); + clk_bulk_disable_unprepare(gpu->nr_clocks, gpu->grp_clks); /* * Set the clock to a deliberately low rate. On older targets the clock @@ -785,44 +767,22 @@ static irqreturn_t irq_handler(int irq, void *data) return gpu->funcs->irq(gpu); } -static struct clk *get_clock(struct device *dev, const char *name) -{ - struct clk *clk = devm_clk_get(dev, name); - - return IS_ERR(clk) ? NULL : clk; -} - static int get_clocks(struct platform_device *pdev, struct msm_gpu *gpu) { - struct device *dev = &pdev->dev; - struct property *prop; - const char *name; - int i = 0; + int ret = msm_clk_bulk_get(&pdev->dev, &gpu->grp_clks); - gpu->nr_clocks = of_property_count_strings(dev->of_node, "clock-names"); - if (gpu->nr_clocks < 1) { + if (ret < 1) { gpu->nr_clocks = 0; - return 0; - } - - gpu->grp_clks = devm_kcalloc(dev, sizeof(struct clk *), gpu->nr_clocks, - GFP_KERNEL); - if (!gpu->grp_clks) { - gpu->nr_clocks = 0; - return -ENOMEM; + return ret; } - of_property_for_each_string(dev->of_node, "clock-names", prop, name) { - gpu->grp_clks[i] = get_clock(dev, name); + gpu->nr_clocks = ret; - /* Remember the key clocks that we need to control later */ - if (!strcmp(name, "core") || !strcmp(name, "core_clk")) - gpu->core_clk = gpu->grp_clks[i]; - else if (!strcmp(name, "rbbmtimer") || !strcmp(name, "rbbmtimer_clk")) - gpu->rbbmtimer_clk = gpu->grp_clks[i]; + gpu->core_clk = msm_clk_bulk_get_clock(gpu->grp_clks, + gpu->nr_clocks, "core"); - ++i; - } + gpu->rbbmtimer_clk = msm_clk_bulk_get_clock(gpu->grp_clks, + gpu->nr_clocks, "rbbmtimer"); return 0; } diff --git a/drivers/gpu/drm/msm/msm_gpu.h b/drivers/gpu/drm/msm/msm_gpu.h index 1c6105bc55c7..9122ee6e55e4 100644 --- a/drivers/gpu/drm/msm/msm_gpu.h +++ b/drivers/gpu/drm/msm/msm_gpu.h @@ -112,7 +112,7 @@ struct msm_gpu { /* Power Control: */ struct regulator *gpu_reg, *gpu_cx; - struct clk **grp_clks; + struct clk_bulk_data *grp_clks; int nr_clocks; struct clk *ebi1_clk, *core_clk, *rbbmtimer_clk; uint32_t fast_rate; diff --git a/drivers/hwmon/ibmpowernv.c b/drivers/hwmon/ibmpowernv.c index f829dadfd5a0..83472808c816 100644 --- a/drivers/hwmon/ibmpowernv.c +++ b/drivers/hwmon/ibmpowernv.c @@ -90,11 +90,20 @@ struct sensor_data { char label[MAX_LABEL_LEN]; char name[MAX_ATTR_LEN]; struct device_attribute dev_attr; + struct sensor_group_data *sgrp_data; +}; + +struct sensor_group_data { + struct mutex mutex; + u32 gid; + bool enable; }; struct platform_data { const struct attribute_group *attr_groups[MAX_SENSOR_TYPE + 1]; + struct sensor_group_data *sgrp_data; u32 sensors_count; /* Total count of sensors from each group */ + u32 nr_sensor_groups; /* Total number of sensor groups */ }; static ssize_t show_sensor(struct device *dev, struct device_attribute *devattr, @@ -105,6 +114,9 @@ static ssize_t show_sensor(struct device *dev, struct device_attribute *devattr, ssize_t ret; u64 x; + if (sdata->sgrp_data && !sdata->sgrp_data->enable) + return -ENODATA; + ret = opal_get_sensor_data_u64(sdata->id, &x); if (ret) @@ -120,6 +132,46 @@ static ssize_t show_sensor(struct device *dev, struct device_attribute *devattr, return sprintf(buf, "%llu\n", x); } +static ssize_t show_enable(struct device *dev, + struct device_attribute *devattr, char *buf) +{ + struct sensor_data *sdata = container_of(devattr, struct sensor_data, + dev_attr); + + return sprintf(buf, "%u\n", sdata->sgrp_data->enable); +} + +static ssize_t store_enable(struct device *dev, + struct device_attribute *devattr, + const char *buf, size_t count) +{ + struct sensor_data *sdata = container_of(devattr, struct sensor_data, + dev_attr); + struct sensor_group_data *sgrp_data = sdata->sgrp_data; + int ret; + bool data; + + ret = kstrtobool(buf, &data); + if (ret) + return ret; + + ret = mutex_lock_interruptible(&sgrp_data->mutex); + if (ret) + return ret; + + if (data != sgrp_data->enable) { + ret = sensor_group_enable(sgrp_data->gid, data); + if (!ret) + sgrp_data->enable = data; + } + + if (!ret) + ret = count; + + mutex_unlock(&sgrp_data->mutex); + return ret; +} + static ssize_t show_label(struct device *dev, struct device_attribute *devattr, char *buf) { @@ -292,12 +344,115 @@ static u32 get_sensor_hwmon_index(struct sensor_data *sdata, return ++sensor_groups[sdata->type].hwmon_index; } +static int init_sensor_group_data(struct platform_device *pdev, + struct platform_data *pdata) +{ + struct sensor_group_data *sgrp_data; + struct device_node *groups, *sgrp; + int count = 0, ret = 0; + enum sensors type; + + groups = of_find_compatible_node(NULL, NULL, "ibm,opal-sensor-group"); + if (!groups) + return ret; + + for_each_child_of_node(groups, sgrp) { + type = get_sensor_type(sgrp); + if (type != MAX_SENSOR_TYPE) + pdata->nr_sensor_groups++; + } + + if (!pdata->nr_sensor_groups) + goto out; + + sgrp_data = devm_kcalloc(&pdev->dev, pdata->nr_sensor_groups, + sizeof(*sgrp_data), GFP_KERNEL); + if (!sgrp_data) { + ret = -ENOMEM; + goto out; + } + + for_each_child_of_node(groups, sgrp) { + u32 gid; + + type = get_sensor_type(sgrp); + if (type == MAX_SENSOR_TYPE) + continue; + + if (of_property_read_u32(sgrp, "sensor-group-id", &gid)) + continue; + + if (of_count_phandle_with_args(sgrp, "sensors", NULL) <= 0) + continue; + + sensor_groups[type].attr_count++; + sgrp_data[count].gid = gid; + mutex_init(&sgrp_data[count].mutex); + sgrp_data[count++].enable = false; + } + + pdata->sgrp_data = sgrp_data; +out: + of_node_put(groups); + return ret; +} + +static struct sensor_group_data *get_sensor_group(struct platform_data *pdata, + struct device_node *node, + enum sensors gtype) +{ + struct sensor_group_data *sgrp_data = pdata->sgrp_data; + struct device_node *groups, *sgrp; + + groups = of_find_compatible_node(NULL, NULL, "ibm,opal-sensor-group"); + if (!groups) + return NULL; + + for_each_child_of_node(groups, sgrp) { + struct of_phandle_iterator it; + u32 gid; + int rc, i; + enum sensors type; + + type = get_sensor_type(sgrp); + if (type != gtype) + continue; + + if (of_property_read_u32(sgrp, "sensor-group-id", &gid)) + continue; + + of_for_each_phandle(&it, rc, sgrp, "sensors", NULL, 0) + if (it.phandle == node->phandle) { + of_node_put(it.node); + break; + } + + if (rc) + continue; + + for (i = 0; i < pdata->nr_sensor_groups; i++) + if (gid == sgrp_data[i].gid) { + of_node_put(sgrp); + of_node_put(groups); + return &sgrp_data[i]; + } + } + + of_node_put(groups); + return NULL; +} + static int populate_attr_groups(struct platform_device *pdev) { struct platform_data *pdata = platform_get_drvdata(pdev); const struct attribute_group **pgroups = pdata->attr_groups; struct device_node *opal, *np; enum sensors type; + int ret; + + ret = init_sensor_group_data(pdev, pdata); + if (ret) + return ret; opal = of_find_node_by_path("/ibm,opal/sensors"); for_each_child_of_node(opal, np) { @@ -344,7 +499,10 @@ static int populate_attr_groups(struct platform_device *pdev) static void create_hwmon_attr(struct sensor_data *sdata, const char *attr_name, ssize_t (*show)(struct device *dev, struct device_attribute *attr, - char *buf)) + char *buf), + ssize_t (*store)(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count)) { snprintf(sdata->name, MAX_ATTR_LEN, "%s%d_%s", sensor_groups[sdata->type].name, sdata->hwmon_index, @@ -352,23 +510,33 @@ static void create_hwmon_attr(struct sensor_data *sdata, const char *attr_name, sysfs_attr_init(&sdata->dev_attr.attr); sdata->dev_attr.attr.name = sdata->name; - sdata->dev_attr.attr.mode = S_IRUGO; sdata->dev_attr.show = show; + if (store) { + sdata->dev_attr.store = store; + sdata->dev_attr.attr.mode = 0664; + } else { + sdata->dev_attr.attr.mode = 0444; + } } static void populate_sensor(struct sensor_data *sdata, int od, int hd, int sid, const char *attr_name, enum sensors type, const struct attribute_group *pgroup, + struct sensor_group_data *sgrp_data, ssize_t (*show)(struct device *dev, struct device_attribute *attr, - char *buf)) + char *buf), + ssize_t (*store)(struct device *dev, + struct device_attribute *attr, + const char *buf, size_t count)) { sdata->id = sid; sdata->type = type; sdata->opal_index = od; sdata->hwmon_index = hd; - create_hwmon_attr(sdata, attr_name, show); + create_hwmon_attr(sdata, attr_name, show, store); pgroup->attrs[sensor_groups[type].attr_count++] = &sdata->dev_attr.attr; + sdata->sgrp_data = sgrp_data; } static char *get_max_attr(enum sensors type) @@ -403,24 +571,23 @@ static int create_device_attrs(struct platform_device *pdev) const struct attribute_group **pgroups = pdata->attr_groups; struct device_node *opal, *np; struct sensor_data *sdata; - u32 sensor_id; - enum sensors type; u32 count = 0; - int err = 0; + u32 group_attr_id[MAX_SENSOR_TYPE] = {0}; - opal = of_find_node_by_path("/ibm,opal/sensors"); sdata = devm_kcalloc(&pdev->dev, pdata->sensors_count, sizeof(*sdata), GFP_KERNEL); - if (!sdata) { - err = -ENOMEM; - goto exit_put_node; - } + if (!sdata) + return -ENOMEM; + opal = of_find_node_by_path("/ibm,opal/sensors"); for_each_child_of_node(opal, np) { + struct sensor_group_data *sgrp_data; const char *attr_name; - u32 opal_index; + u32 opal_index, hw_id; + u32 sensor_id; const char *label; + enum sensors type; if (np->name == NULL) continue; @@ -456,14 +623,12 @@ static int create_device_attrs(struct platform_device *pdev) opal_index = INVALID_INDEX; } - sdata[count].opal_index = opal_index; - sdata[count].hwmon_index = - get_sensor_hwmon_index(&sdata[count], sdata, count); - - create_hwmon_attr(&sdata[count], attr_name, show_sensor); - - pgroups[type]->attrs[sensor_groups[type].attr_count++] = - &sdata[count++].dev_attr.attr; + hw_id = get_sensor_hwmon_index(&sdata[count], sdata, count); + sgrp_data = get_sensor_group(pdata, np, type); + populate_sensor(&sdata[count], opal_index, hw_id, sensor_id, + attr_name, type, pgroups[type], sgrp_data, + show_sensor, NULL); + count++; if (!of_property_read_string(np, "label", &label)) { /* @@ -474,35 +639,43 @@ static int create_device_attrs(struct platform_device *pdev) */ make_sensor_label(np, &sdata[count], label); - populate_sensor(&sdata[count], opal_index, - sdata[count - 1].hwmon_index, + populate_sensor(&sdata[count], opal_index, hw_id, sensor_id, "label", type, pgroups[type], - show_label); + NULL, show_label, NULL); count++; } if (!of_property_read_u32(np, "sensor-data-max", &sensor_id)) { attr_name = get_max_attr(type); - populate_sensor(&sdata[count], opal_index, - sdata[count - 1].hwmon_index, + populate_sensor(&sdata[count], opal_index, hw_id, sensor_id, attr_name, type, - pgroups[type], show_sensor); + pgroups[type], sgrp_data, show_sensor, + NULL); count++; } if (!of_property_read_u32(np, "sensor-data-min", &sensor_id)) { attr_name = get_min_attr(type); - populate_sensor(&sdata[count], opal_index, - sdata[count - 1].hwmon_index, + populate_sensor(&sdata[count], opal_index, hw_id, sensor_id, attr_name, type, - pgroups[type], show_sensor); + pgroups[type], sgrp_data, show_sensor, + NULL); + count++; + } + + if (sgrp_data && !sgrp_data->enable) { + sgrp_data->enable = true; + hw_id = ++group_attr_id[type]; + populate_sensor(&sdata[count], opal_index, hw_id, + sgrp_data->gid, "enable", type, + pgroups[type], sgrp_data, show_enable, + store_enable); count++; } } -exit_put_node: of_node_put(opal); - return err; + return 0; } static int ibmpowernv_probe(struct platform_device *pdev) @@ -517,6 +690,7 @@ static int ibmpowernv_probe(struct platform_device *pdev) platform_set_drvdata(pdev, pdata); pdata->sensors_count = 0; + pdata->nr_sensor_groups = 0; err = populate_attr_groups(pdev); if (err) return err; diff --git a/drivers/macintosh/Kconfig b/drivers/macintosh/Kconfig index 97a420c11eed..47c350cdfb12 100644 --- a/drivers/macintosh/Kconfig +++ b/drivers/macintosh/Kconfig @@ -39,17 +39,6 @@ config ADB_IOP <http://www.angelfire.com/ca2/dev68k/iopdesc.html> to enable direct support for it, say 'Y' here. -config ADB_PMU68K - bool "Include PMU (Powerbook) ADB driver" - depends on ADB && MAC - help - Say Y here if want your kernel to support the m68k based Powerbooks. - This includes the PowerBook 140, PowerBook 145, PowerBook 150, - PowerBook 160, PowerBook 165, PowerBook 165c, PowerBook 170, - PowerBook 180, PowerBook, 180c, PowerBook 190cs, PowerBook 520, - PowerBook Duo 210, PowerBook Duo 230, PowerBook Duo 250, - PowerBook Duo 270c, PowerBook Duo 280 and PowerBook Duo 280c. - # we want to change this to something like CONFIG_SYSCTRL_CUDA/PMU config ADB_CUDA bool "Support for Cuda/Egret based Macs and PowerMacs" @@ -65,8 +54,8 @@ config ADB_CUDA If unsure say Y. config ADB_PMU - bool "Support for PMU based PowerMacs" - depends on PPC_PMAC + bool "Support for PMU based PowerMacs and PowerBooks" + depends on PPC_PMAC || MAC help On PowerBooks, iBooks, and recent iMacs and Power Macintoshes, the PMU is an embedded microprocessor whose primary function is to @@ -79,7 +68,7 @@ config ADB_PMU config ADB_PMU_LED bool "Support for the Power/iBook front LED" - depends on ADB_PMU + depends on PPC_PMAC && ADB_PMU select NEW_LEDS select LEDS_CLASS help @@ -122,7 +111,7 @@ config PMAC_MEDIABAY config PMAC_BACKLIGHT bool "Backlight control for LCD screens" - depends on ADB_PMU && FB = y && (BROKEN || !PPC64) + depends on PPC_PMAC && ADB_PMU && FB = y && (BROKEN || !PPC64) select FB_BACKLIGHT help Say Y here to enable Macintosh specific extensions of the generic diff --git a/drivers/macintosh/Makefile b/drivers/macintosh/Makefile index ee803638e595..49819b1b6f20 100644 --- a/drivers/macintosh/Makefile +++ b/drivers/macintosh/Makefile @@ -22,7 +22,6 @@ obj-$(CONFIG_PMAC_SMU) += smu.o obj-$(CONFIG_ADB) += adb.o obj-$(CONFIG_ADB_MACII) += via-macii.o obj-$(CONFIG_ADB_IOP) += adb-iop.o -obj-$(CONFIG_ADB_PMU68K) += via-pmu68k.o obj-$(CONFIG_ADB_MACIO) += macio-adb.o obj-$(CONFIG_THERM_WINDTUNNEL) += therm_windtunnel.o diff --git a/drivers/macintosh/adb.c b/drivers/macintosh/adb.c index 4c8097e0e6fe..76e98f0f7a3e 100644 --- a/drivers/macintosh/adb.c +++ b/drivers/macintosh/adb.c @@ -65,7 +65,7 @@ static struct adb_driver *adb_driver_list[] = { #ifdef CONFIG_ADB_IOP &adb_iop_driver, #endif -#if defined(CONFIG_ADB_PMU) || defined(CONFIG_ADB_PMU68K) +#ifdef CONFIG_ADB_PMU &via_pmu_driver, #endif #ifdef CONFIG_ADB_MACIO diff --git a/drivers/macintosh/via-pmu.c b/drivers/macintosh/via-pmu.c index 25c1ce811053..d72c450aebe5 100644 --- a/drivers/macintosh/via-pmu.c +++ b/drivers/macintosh/via-pmu.c @@ -1,6 +1,6 @@ // SPDX-License-Identifier: GPL-2.0 /* - * Device driver for the via-pmu on Apple Powermacs. + * Device driver for the PMU in Apple PowerBooks and PowerMacs. * * The VIA (versatile interface adapter) interfaces to the PMU, * a 6805 microprocessor core whose primary function is to control @@ -49,20 +49,26 @@ #include <linux/compat.h> #include <linux/of_address.h> #include <linux/of_irq.h> -#include <asm/prom.h> +#include <linux/uaccess.h> #include <asm/machdep.h> #include <asm/io.h> #include <asm/pgtable.h> #include <asm/sections.h> #include <asm/irq.h> +#ifdef CONFIG_PPC_PMAC #include <asm/pmac_feature.h> #include <asm/pmac_pfunc.h> #include <asm/pmac_low_i2c.h> -#include <linux/uaccess.h> +#include <asm/prom.h> #include <asm/mmu_context.h> #include <asm/cputable.h> #include <asm/time.h> #include <asm/backlight.h> +#else +#include <asm/macintosh.h> +#include <asm/macints.h> +#include <asm/mac_via.h> +#endif #include "via-pmu-event.h" @@ -76,7 +82,6 @@ #define BATTERY_POLLING_COUNT 2 static DEFINE_MUTEX(pmu_info_proc_mutex); -static volatile unsigned char __iomem *via; /* VIA registers - spaced 0x200 bytes apart */ #define RS 0x200 /* skip between registers */ @@ -98,8 +103,13 @@ static volatile unsigned char __iomem *via; #define ANH (15*RS) /* A-side data, no handshake */ /* Bits in B data register: both active low */ +#ifdef CONFIG_PPC_PMAC #define TACK 0x08 /* Transfer acknowledge (input) */ #define TREQ 0x10 /* Transfer request (output) */ +#else +#define TACK 0x02 +#define TREQ 0x04 +#endif /* Bits in ACR */ #define SR_CTRL 0x1c /* Shift register control bits */ @@ -114,6 +124,7 @@ static volatile unsigned char __iomem *via; #define CB1_INT 0x10 /* transition on CB1 input */ static volatile enum pmu_state { + uninitialized = 0, idle, sending, intack, @@ -140,11 +151,15 @@ static int data_index; static int data_len; static volatile int adb_int_pending; static volatile int disable_poll; -static struct device_node *vias; static int pmu_kind = PMU_UNKNOWN; static int pmu_fully_inited; static int pmu_has_adb; +#ifdef CONFIG_PPC_PMAC +static volatile unsigned char __iomem *via1; +static volatile unsigned char __iomem *via2; +static struct device_node *vias; static struct device_node *gpio_node; +#endif static unsigned char __iomem *gpio_reg; static int gpio_irq = 0; static int gpio_irq_enabled = -1; @@ -157,7 +172,9 @@ static int drop_interrupts; static int option_lid_wakeup = 1; #endif /* CONFIG_SUSPEND && CONFIG_PPC32 */ static unsigned long async_req_locks; -static unsigned int pmu_irq_stats[11]; + +#define NUM_IRQ_STATS 13 +static unsigned int pmu_irq_stats[NUM_IRQ_STATS]; static struct proc_dir_entry *proc_pmu_root; static struct proc_dir_entry *proc_pmu_info; @@ -271,10 +288,11 @@ static char *pbook_type[] = { int __init find_via_pmu(void) { +#ifdef CONFIG_PPC_PMAC u64 taddr; const u32 *reg; - if (via) + if (pmu_state != uninitialized) return 1; vias = of_find_node_by_name(NULL, "via-pmu"); if (vias == NULL) @@ -339,50 +357,70 @@ int __init find_via_pmu(void) } else pmu_kind = PMU_UNKNOWN; - via = ioremap(taddr, 0x2000); - if (via == NULL) { + via1 = via2 = ioremap(taddr, 0x2000); + if (via1 == NULL) { printk(KERN_ERR "via-pmu: Can't map address !\n"); goto fail_via_remap; } - out_8(&via[IER], IER_CLR | 0x7f); /* disable all intrs */ - out_8(&via[IFR], 0x7f); /* clear IFR */ + out_8(&via1[IER], IER_CLR | 0x7f); /* disable all intrs */ + out_8(&via1[IFR], 0x7f); /* clear IFR */ pmu_state = idle; if (!init_pmu()) goto fail_init; - printk(KERN_INFO "PMU driver v%d initialized for %s, firmware: %02x\n", - PMU_DRIVER_VERSION, pbook_type[pmu_kind], pmu_version); - sys_ctrler = SYS_CTRLER_PMU; return 1; fail_init: - iounmap(via); - via = NULL; + iounmap(via1); + via1 = via2 = NULL; fail_via_remap: iounmap(gpio_reg); gpio_reg = NULL; fail: of_node_put(vias); vias = NULL; + pmu_state = uninitialized; return 0; +#else + if (macintosh_config->adb_type != MAC_ADB_PB2) + return 0; + + pmu_kind = PMU_UNKNOWN; + + spin_lock_init(&pmu_lock); + + pmu_has_adb = 1; + + pmu_intr_mask = PMU_INT_PCEJECT | + PMU_INT_SNDBRT | + PMU_INT_ADB | + PMU_INT_TICK; + + pmu_state = idle; + + if (!init_pmu()) { + pmu_state = uninitialized; + return 0; + } + + return 1; +#endif /* !CONFIG_PPC_PMAC */ } #ifdef CONFIG_ADB static int pmu_probe(void) { - return vias == NULL? -ENODEV: 0; + return pmu_state == uninitialized ? -ENODEV : 0; } -static int __init pmu_init(void) +static int pmu_init(void) { - if (vias == NULL) - return -ENODEV; - return 0; + return pmu_state == uninitialized ? -ENODEV : 0; } #endif /* CONFIG_ADB */ @@ -395,13 +433,14 @@ static int __init pmu_init(void) */ static int __init via_pmu_start(void) { - unsigned int irq; + unsigned int __maybe_unused irq; - if (vias == NULL) + if (pmu_state == uninitialized) return -ENODEV; batt_req.complete = 1; +#ifdef CONFIG_PPC_PMAC irq = irq_of_parse_and_map(vias, 0); if (!irq) { printk(KERN_ERR "via-pmu: can't map interrupt\n"); @@ -437,7 +476,20 @@ static int __init via_pmu_start(void) } /* Enable interrupts */ - out_8(&via[IER], IER_SET | SR_INT | CB1_INT); + out_8(&via1[IER], IER_SET | SR_INT | CB1_INT); +#else + if (request_irq(IRQ_MAC_ADB_SR, via_pmu_interrupt, IRQF_NO_SUSPEND, + "VIA-PMU-SR", NULL)) { + pr_err("%s: couldn't get SR irq\n", __func__); + return -ENODEV; + } + if (request_irq(IRQ_MAC_ADB_CL, via_pmu_interrupt, IRQF_NO_SUSPEND, + "VIA-PMU-CL", NULL)) { + pr_err("%s: couldn't get CL irq\n", __func__); + free_irq(IRQ_MAC_ADB_SR, NULL); + return -ENODEV; + } +#endif /* !CONFIG_PPC_PMAC */ pmu_fully_inited = 1; @@ -463,7 +515,7 @@ arch_initcall(via_pmu_start); */ static int __init via_pmu_dev_init(void) { - if (vias == NULL) + if (pmu_state == uninitialized) return -ENODEV; #ifdef CONFIG_PMAC_BACKLIGHT @@ -534,8 +586,9 @@ init_pmu(void) int timeout; struct adb_request req; - out_8(&via[B], via[B] | TREQ); /* negate TREQ */ - out_8(&via[DIRB], (via[DIRB] | TREQ) & ~TACK); /* TACK in, TREQ out */ + /* Negate TREQ. Set TACK to input and TREQ to output. */ + out_8(&via2[B], in_8(&via2[B]) | TREQ); + out_8(&via2[DIRB], (in_8(&via2[DIRB]) | TREQ) & ~TACK); pmu_request(&req, NULL, 2, PMU_SET_INTR_MASK, pmu_intr_mask); timeout = 100000; @@ -587,6 +640,10 @@ init_pmu(void) option_server_mode ? "enabled" : "disabled"); } } + + printk(KERN_INFO "PMU driver v%d initialized for %s, firmware: %02x\n", + PMU_DRIVER_VERSION, pbook_type[pmu_kind], pmu_version); + return 1; } @@ -625,6 +682,7 @@ static void pmu_set_server_mode(int server_mode) static void done_battery_state_ohare(struct adb_request* req) { +#ifdef CONFIG_PPC_PMAC /* format: * [0] : flags * 0x01 : AC indicator @@ -706,6 +764,7 @@ done_battery_state_ohare(struct adb_request* req) pmu_batteries[pmu_cur_battery].amperage = amperage; pmu_batteries[pmu_cur_battery].voltage = voltage; pmu_batteries[pmu_cur_battery].time_remaining = time; +#endif /* CONFIG_PPC_PMAC */ clear_bit(0, &async_req_locks); } @@ -816,9 +875,9 @@ static int pmu_info_proc_show(struct seq_file *m, void *v) static int pmu_irqstats_proc_show(struct seq_file *m, void *v) { int i; - static const char *irq_names[] = { - "Total CB1 triggered events", - "Total GPIO1 triggered events", + static const char *irq_names[NUM_IRQ_STATS] = { + "Unknown interrupt (type 0)", + "Unknown interrupt (type 1)", "PC-Card eject button", "Sound/Brightness button", "ADB message", @@ -827,10 +886,12 @@ static int pmu_irqstats_proc_show(struct seq_file *m, void *v) "Tick timer", "Ghost interrupt (zero len)", "Empty interrupt (empty mask)", - "Max irqs in a row" + "Max irqs in a row", + "Total CB1 triggered events", + "Total GPIO1 triggered events", }; - for (i=0; i<11; i++) { + for (i = 0; i < NUM_IRQ_STATS; i++) { seq_printf(m, " %2u: %10u (%s)\n", i, pmu_irq_stats[i], irq_names[i]); } @@ -928,7 +989,7 @@ static int pmu_send_request(struct adb_request *req, int sync) { int i, ret; - if ((vias == NULL) || (!pmu_fully_inited)) { + if (pmu_state == uninitialized || !pmu_fully_inited) { req->complete = 1; return -ENXIO; } @@ -1022,7 +1083,7 @@ static int __pmu_adb_autopoll(int devs) static int pmu_adb_autopoll(int devs) { - if ((vias == NULL) || (!pmu_fully_inited) || !pmu_has_adb) + if (pmu_state == uninitialized || !pmu_fully_inited || !pmu_has_adb) return -ENXIO; adb_dev_map = devs; @@ -1035,7 +1096,7 @@ static int pmu_adb_reset_bus(void) struct adb_request req; int save_autopoll = adb_dev_map; - if ((vias == NULL) || (!pmu_fully_inited) || !pmu_has_adb) + if (pmu_state == uninitialized || !pmu_fully_inited || !pmu_has_adb) return -ENXIO; /* anyone got a better idea?? */ @@ -1071,7 +1132,7 @@ pmu_request(struct adb_request *req, void (*done)(struct adb_request *), va_list list; int i; - if (vias == NULL) + if (pmu_state == uninitialized) return -ENXIO; if (nbytes < 0 || nbytes > 32) { @@ -1096,7 +1157,7 @@ pmu_queue_request(struct adb_request *req) unsigned long flags; int nsend; - if (via == NULL) { + if (pmu_state == uninitialized) { req->complete = 1; return -ENXIO; } @@ -1136,7 +1197,7 @@ wait_for_ack(void) * reported */ int timeout = 4000; - while ((in_8(&via[B]) & TACK) == 0) { + while ((in_8(&via2[B]) & TACK) == 0) { if (--timeout < 0) { printk(KERN_ERR "PMU not responding (!ack)\n"); return; @@ -1150,23 +1211,19 @@ wait_for_ack(void) static inline void send_byte(int x) { - volatile unsigned char __iomem *v = via; - - out_8(&v[ACR], in_8(&v[ACR]) | SR_OUT | SR_EXT); - out_8(&v[SR], x); - out_8(&v[B], in_8(&v[B]) & ~TREQ); /* assert TREQ */ - (void)in_8(&v[B]); + out_8(&via1[ACR], in_8(&via1[ACR]) | SR_OUT | SR_EXT); + out_8(&via1[SR], x); + out_8(&via2[B], in_8(&via2[B]) & ~TREQ); /* assert TREQ */ + (void)in_8(&via2[B]); } static inline void recv_byte(void) { - volatile unsigned char __iomem *v = via; - - out_8(&v[ACR], (in_8(&v[ACR]) & ~SR_OUT) | SR_EXT); - in_8(&v[SR]); /* resets SR */ - out_8(&v[B], in_8(&v[B]) & ~TREQ); - (void)in_8(&v[B]); + out_8(&via1[ACR], (in_8(&via1[ACR]) & ~SR_OUT) | SR_EXT); + in_8(&via1[SR]); /* resets SR */ + out_8(&via2[B], in_8(&via2[B]) & ~TREQ); + (void)in_8(&via2[B]); } static inline void @@ -1209,7 +1266,7 @@ pmu_start(void) void pmu_poll(void) { - if (!via) + if (pmu_state == uninitialized) return; if (disable_poll) return; @@ -1219,7 +1276,7 @@ pmu_poll(void) void pmu_poll_adb(void) { - if (!via) + if (pmu_state == uninitialized) return; if (disable_poll) return; @@ -1234,7 +1291,7 @@ pmu_poll_adb(void) void pmu_wait_complete(struct adb_request *req) { - if (!via) + if (pmu_state == uninitialized) return; while((pmu_state != idle && pmu_state != locked) || !req->complete) via_pmu_interrupt(0, NULL); @@ -1250,7 +1307,7 @@ pmu_suspend(void) { unsigned long flags; - if (!via) + if (pmu_state == uninitialized) return; spin_lock_irqsave(&pmu_lock, flags); @@ -1269,7 +1326,7 @@ pmu_suspend(void) if (!adb_int_pending && pmu_state == idle && !req_awaiting_reply) { if (gpio_irq >= 0) disable_irq_nosync(gpio_irq); - out_8(&via[IER], CB1_INT | IER_CLR); + out_8(&via1[IER], CB1_INT | IER_CLR); spin_unlock_irqrestore(&pmu_lock, flags); break; } @@ -1281,7 +1338,7 @@ pmu_resume(void) { unsigned long flags; - if (!via || (pmu_suspended < 1)) + if (pmu_state == uninitialized || pmu_suspended < 1) return; spin_lock_irqsave(&pmu_lock, flags); @@ -1293,7 +1350,7 @@ pmu_resume(void) adb_int_pending = 1; if (gpio_irq >= 0) enable_irq(gpio_irq); - out_8(&via[IER], CB1_INT | IER_SET); + out_8(&via1[IER], CB1_INT | IER_SET); spin_unlock_irqrestore(&pmu_lock, flags); pmu_poll(); } @@ -1302,7 +1359,8 @@ pmu_resume(void) static void pmu_handle_data(unsigned char *data, int len) { - unsigned char ints, pirq; + unsigned char ints; + int idx; int i = 0; asleep = 0; @@ -1324,25 +1382,24 @@ pmu_handle_data(unsigned char *data, int len) ints &= ~(PMU_INT_ADB_AUTO | PMU_INT_AUTO_SRQ_POLL); next: - if (ints == 0) { if (i > pmu_irq_stats[10]) pmu_irq_stats[10] = i; return; } - - for (pirq = 0; pirq < 8; pirq++) - if (ints & (1 << pirq)) - break; - pmu_irq_stats[pirq]++; i++; - ints &= ~(1 << pirq); + + idx = ffs(ints) - 1; + ints &= ~BIT(idx); + + pmu_irq_stats[idx]++; /* Note: for some reason, we get an interrupt with len=1, * data[0]==0 after each normal ADB interrupt, at least * on the Pismo. Still investigating... --BenH */ - if ((1 << pirq) & PMU_INT_ADB) { + switch (BIT(idx)) { + case PMU_INT_ADB: if ((data[0] & PMU_INT_ADB_AUTO) == 0) { struct adb_request *req = req_awaiting_reply; if (!req) { @@ -1358,6 +1415,7 @@ next: } pmu_done(req); } else { +#ifdef CONFIG_XMON if (len == 4 && data[1] == 0x2c) { extern int xmon_wants_key, xmon_adb_keycode; if (xmon_wants_key) { @@ -1365,6 +1423,7 @@ next: return; } } +#endif /* CONFIG_XMON */ #ifdef CONFIG_ADB /* * XXX On the [23]400 the PMU gives us an up @@ -1378,25 +1437,28 @@ next: adb_input(data+1, len-1, 1); #endif /* CONFIG_ADB */ } - } + break; + /* Sound/brightness button pressed */ - else if ((1 << pirq) & PMU_INT_SNDBRT) { + case PMU_INT_SNDBRT: #ifdef CONFIG_PMAC_BACKLIGHT if (len == 3) pmac_backlight_set_legacy_brightness_pmu(data[1] >> 4); #endif - } + break; + /* Tick interrupt */ - else if ((1 << pirq) & PMU_INT_TICK) { - /* Environement or tick interrupt, query batteries */ + case PMU_INT_TICK: + /* Environment or tick interrupt, query batteries */ if (pmu_battery_count) { if ((--query_batt_timer) == 0) { query_battery_state(); query_batt_timer = BATTERY_POLLING_COUNT; } } - } - else if ((1 << pirq) & PMU_INT_ENVIRONMENT) { + break; + + case PMU_INT_ENVIRONMENT: if (pmu_battery_count) query_battery_state(); pmu_pass_intr(data, len); @@ -1406,7 +1468,9 @@ next: via_pmu_event(PMU_EVT_POWER, !!(data[1]&8)); via_pmu_event(PMU_EVT_LID, data[1]&1); } - } else { + break; + + default: pmu_pass_intr(data, len); } goto next; @@ -1418,21 +1482,20 @@ pmu_sr_intr(void) struct adb_request *req; int bite = 0; - if (via[B] & TREQ) { - printk(KERN_ERR "PMU: spurious SR intr (%x)\n", via[B]); - out_8(&via[IFR], SR_INT); + if (in_8(&via2[B]) & TREQ) { + printk(KERN_ERR "PMU: spurious SR intr (%x)\n", in_8(&via2[B])); return NULL; } /* The ack may not yet be low when we get the interrupt */ - while ((in_8(&via[B]) & TACK) != 0) + while ((in_8(&via2[B]) & TACK) != 0) ; /* if reading grab the byte, and reset the interrupt */ if (pmu_state == reading || pmu_state == reading_intr) - bite = in_8(&via[SR]); + bite = in_8(&via1[SR]); /* reset TREQ and wait for TACK to go high */ - out_8(&via[B], in_8(&via[B]) | TREQ); + out_8(&via2[B], in_8(&via2[B]) | TREQ); wait_for_ack(); switch (pmu_state) { @@ -1533,26 +1596,46 @@ via_pmu_interrupt(int irq, void *arg) ++disable_poll; for (;;) { - intr = in_8(&via[IFR]) & (SR_INT | CB1_INT); + /* On 68k Macs, VIA interrupts are dispatched individually. + * Unless we are polling, the relevant IRQ flag has already + * been cleared. + */ + intr = 0; + if (IS_ENABLED(CONFIG_PPC_PMAC) || !irq) { + intr = in_8(&via1[IFR]) & (SR_INT | CB1_INT); + out_8(&via1[IFR], intr); + } +#ifndef CONFIG_PPC_PMAC + switch (irq) { + case IRQ_MAC_ADB_CL: + intr = CB1_INT; + break; + case IRQ_MAC_ADB_SR: + intr = SR_INT; + break; + } +#endif if (intr == 0) break; handled = 1; if (++nloop > 1000) { printk(KERN_DEBUG "PMU: stuck in intr loop, " "intr=%x, ier=%x pmu_state=%d\n", - intr, in_8(&via[IER]), pmu_state); + intr, in_8(&via1[IER]), pmu_state); break; } - out_8(&via[IFR], intr); if (intr & CB1_INT) { adb_int_pending = 1; - pmu_irq_stats[0]++; + pmu_irq_stats[11]++; } if (intr & SR_INT) { req = pmu_sr_intr(); if (req) break; } +#ifndef CONFIG_PPC_PMAC + break; +#endif } recheck: @@ -1619,7 +1702,7 @@ pmu_unlock(void) } -static irqreturn_t +static __maybe_unused irqreturn_t gpio1_interrupt(int irq, void *arg) { unsigned long flags; @@ -1630,7 +1713,7 @@ gpio1_interrupt(int irq, void *arg) disable_irq_nosync(gpio_irq); gpio_irq_enabled = 0; } - pmu_irq_stats[1]++; + pmu_irq_stats[12]++; adb_int_pending = 1; spin_unlock_irqrestore(&pmu_lock, flags); via_pmu_interrupt(0, NULL); @@ -1644,7 +1727,7 @@ pmu_enable_irled(int on) { struct adb_request req; - if (vias == NULL) + if (pmu_state == uninitialized) return ; if (pmu_kind == PMU_KEYLARGO_BASED) return ; @@ -1659,7 +1742,7 @@ pmu_restart(void) { struct adb_request req; - if (via == NULL) + if (pmu_state == uninitialized) return; local_irq_disable(); @@ -1684,7 +1767,7 @@ pmu_shutdown(void) { struct adb_request req; - if (via == NULL) + if (pmu_state == uninitialized) return; local_irq_disable(); @@ -1712,7 +1795,7 @@ pmu_shutdown(void) int pmu_present(void) { - return via != NULL; + return pmu_state != uninitialized; } #if defined(CONFIG_SUSPEND) && defined(CONFIG_PPC32) @@ -1725,29 +1808,29 @@ static u32 save_via[8]; static void save_via_state(void) { - save_via[0] = in_8(&via[ANH]); - save_via[1] = in_8(&via[DIRA]); - save_via[2] = in_8(&via[B]); - save_via[3] = in_8(&via[DIRB]); - save_via[4] = in_8(&via[PCR]); - save_via[5] = in_8(&via[ACR]); - save_via[6] = in_8(&via[T1CL]); - save_via[7] = in_8(&via[T1CH]); + save_via[0] = in_8(&via1[ANH]); + save_via[1] = in_8(&via1[DIRA]); + save_via[2] = in_8(&via1[B]); + save_via[3] = in_8(&via1[DIRB]); + save_via[4] = in_8(&via1[PCR]); + save_via[5] = in_8(&via1[ACR]); + save_via[6] = in_8(&via1[T1CL]); + save_via[7] = in_8(&via1[T1CH]); } static void restore_via_state(void) { - out_8(&via[ANH], save_via[0]); - out_8(&via[DIRA], save_via[1]); - out_8(&via[B], save_via[2]); - out_8(&via[DIRB], save_via[3]); - out_8(&via[PCR], save_via[4]); - out_8(&via[ACR], save_via[5]); - out_8(&via[T1CL], save_via[6]); - out_8(&via[T1CH], save_via[7]); - out_8(&via[IER], IER_CLR | 0x7f); /* disable all intrs */ - out_8(&via[IFR], 0x7f); /* clear IFR */ - out_8(&via[IER], IER_SET | SR_INT | CB1_INT); + out_8(&via1[ANH], save_via[0]); + out_8(&via1[DIRA], save_via[1]); + out_8(&via1[B], save_via[2]); + out_8(&via1[DIRB], save_via[3]); + out_8(&via1[PCR], save_via[4]); + out_8(&via1[ACR], save_via[5]); + out_8(&via1[T1CL], save_via[6]); + out_8(&via1[T1CH], save_via[7]); + out_8(&via1[IER], IER_CLR | 0x7f); /* disable all intrs */ + out_8(&via1[IFR], 0x7f); /* clear IFR */ + out_8(&via1[IER], IER_SET | SR_INT | CB1_INT); } #define GRACKLE_PM (1<<7) @@ -2253,6 +2336,7 @@ static int pmu_ioctl(struct file *filp, int error = -EINVAL; switch (cmd) { +#ifdef CONFIG_PPC_PMAC case PMU_IOC_SLEEP: if (!capable(CAP_SYS_ADMIN)) return -EACCES; @@ -2262,6 +2346,7 @@ static int pmu_ioctl(struct file *filp, return put_user(0, argp); else return put_user(1, argp); +#endif #ifdef CONFIG_PMAC_BACKLIGHT_LEGACY /* Compatibility ioctl's for backlight */ @@ -2378,7 +2463,7 @@ static struct miscdevice pmu_device = { static int pmu_device_init(void) { - if (!via) + if (pmu_state == uninitialized) return 0; if (misc_register(&pmu_device) < 0) printk(KERN_ERR "via-pmu: cannot register misc device.\n"); @@ -2389,33 +2474,33 @@ device_initcall(pmu_device_init); #ifdef DEBUG_SLEEP static inline void -polled_handshake(volatile unsigned char __iomem *via) +polled_handshake(void) { - via[B] &= ~TREQ; eieio(); - while ((via[B] & TACK) != 0) + via2[B] &= ~TREQ; eieio(); + while ((via2[B] & TACK) != 0) ; - via[B] |= TREQ; eieio(); - while ((via[B] & TACK) == 0) + via2[B] |= TREQ; eieio(); + while ((via2[B] & TACK) == 0) ; } static inline void -polled_send_byte(volatile unsigned char __iomem *via, int x) +polled_send_byte(int x) { - via[ACR] |= SR_OUT | SR_EXT; eieio(); - via[SR] = x; eieio(); - polled_handshake(via); + via1[ACR] |= SR_OUT | SR_EXT; eieio(); + via1[SR] = x; eieio(); + polled_handshake(); } static inline int -polled_recv_byte(volatile unsigned char __iomem *via) +polled_recv_byte(void) { int x; - via[ACR] = (via[ACR] & ~SR_OUT) | SR_EXT; eieio(); - x = via[SR]; eieio(); - polled_handshake(via); - x = via[SR]; eieio(); + via1[ACR] = (via1[ACR] & ~SR_OUT) | SR_EXT; eieio(); + x = via1[SR]; eieio(); + polled_handshake(); + x = via1[SR]; eieio(); return x; } @@ -2424,7 +2509,6 @@ pmu_polled_request(struct adb_request *req) { unsigned long flags; int i, l, c; - volatile unsigned char __iomem *v = via; req->complete = 1; c = req->data[0]; @@ -2436,21 +2520,21 @@ pmu_polled_request(struct adb_request *req) while (pmu_state != idle) pmu_poll(); - while ((via[B] & TACK) == 0) + while ((via2[B] & TACK) == 0) ; - polled_send_byte(v, c); + polled_send_byte(c); if (l < 0) { l = req->nbytes - 1; - polled_send_byte(v, l); + polled_send_byte(l); } for (i = 1; i <= l; ++i) - polled_send_byte(v, req->data[i]); + polled_send_byte(req->data[i]); l = pmu_data_len[c][1]; if (l < 0) - l = polled_recv_byte(v); + l = polled_recv_byte(); for (i = 0; i < l; ++i) - req->reply[i + req->reply_len] = polled_recv_byte(v); + req->reply[i + req->reply_len] = polled_recv_byte(); if (req->done) (*req->done)(req); diff --git a/drivers/macintosh/via-pmu68k.c b/drivers/macintosh/via-pmu68k.c deleted file mode 100644 index d545ed45e482..000000000000 --- a/drivers/macintosh/via-pmu68k.c +++ /dev/null @@ -1,850 +0,0 @@ -// SPDX-License-Identifier: GPL-2.0 -/* - * Device driver for the PMU on 68K-based Apple PowerBooks - * - * The VIA (versatile interface adapter) interfaces to the PMU, - * a 6805 microprocessor core whose primary function is to control - * battery charging and system power on the PowerBooks. - * The PMU also controls the ADB (Apple Desktop Bus) which connects - * to the keyboard and mouse, as well as the non-volatile RAM - * and the RTC (real time clock) chip. - * - * Adapted for 68K PMU by Joshua M. Thompson - * - * Based largely on the PowerMac PMU code by Paul Mackerras and - * Fabio Riccardi. - * - * Also based on the PMU driver from MkLinux by Apple Computer, Inc. - * and the Open Software Foundation, Inc. - */ - -#include <stdarg.h> -#include <linux/types.h> -#include <linux/errno.h> -#include <linux/kernel.h> -#include <linux/delay.h> -#include <linux/miscdevice.h> -#include <linux/blkdev.h> -#include <linux/pci.h> -#include <linux/init.h> -#include <linux/interrupt.h> - -#include <linux/adb.h> -#include <linux/pmu.h> -#include <linux/cuda.h> - -#include <asm/macintosh.h> -#include <asm/macints.h> -#include <asm/mac_via.h> - -#include <asm/pgtable.h> -#include <asm/irq.h> -#include <linux/uaccess.h> - -/* Misc minor number allocated for /dev/pmu */ -#define PMU_MINOR 154 - -/* VIA registers - spaced 0x200 bytes apart */ -#define RS 0x200 /* skip between registers */ -#define B 0 /* B-side data */ -#define A RS /* A-side data */ -#define DIRB (2*RS) /* B-side direction (1=output) */ -#define DIRA (3*RS) /* A-side direction (1=output) */ -#define T1CL (4*RS) /* Timer 1 ctr/latch (low 8 bits) */ -#define T1CH (5*RS) /* Timer 1 counter (high 8 bits) */ -#define T1LL (6*RS) /* Timer 1 latch (low 8 bits) */ -#define T1LH (7*RS) /* Timer 1 latch (high 8 bits) */ -#define T2CL (8*RS) /* Timer 2 ctr/latch (low 8 bits) */ -#define T2CH (9*RS) /* Timer 2 counter (high 8 bits) */ -#define SR (10*RS) /* Shift register */ -#define ACR (11*RS) /* Auxiliary control register */ -#define PCR (12*RS) /* Peripheral control register */ -#define IFR (13*RS) /* Interrupt flag register */ -#define IER (14*RS) /* Interrupt enable register */ -#define ANH (15*RS) /* A-side data, no handshake */ - -/* Bits in B data register: both active low */ -#define TACK 0x02 /* Transfer acknowledge (input) */ -#define TREQ 0x04 /* Transfer request (output) */ - -/* Bits in ACR */ -#define SR_CTRL 0x1c /* Shift register control bits */ -#define SR_EXT 0x0c /* Shift on external clock */ -#define SR_OUT 0x10 /* Shift out if 1 */ - -/* Bits in IFR and IER */ -#define SR_INT 0x04 /* Shift register full/empty */ -#define CB1_INT 0x10 /* transition on CB1 input */ - -static enum pmu_state { - idle, - sending, - intack, - reading, - reading_intr, -} pmu_state; - -static struct adb_request *current_req; -static struct adb_request *last_req; -static struct adb_request *req_awaiting_reply; -static unsigned char interrupt_data[32]; -static unsigned char *reply_ptr; -static int data_index; -static int data_len; -static int adb_int_pending; -static int pmu_adb_flags; -static int adb_dev_map; -static struct adb_request bright_req_1, bright_req_2, bright_req_3; -static int pmu_kind = PMU_UNKNOWN; -static int pmu_fully_inited; - -int asleep; - -static int pmu_probe(void); -static int pmu_init(void); -static void pmu_start(void); -static irqreturn_t pmu_interrupt(int irq, void *arg); -static int pmu_send_request(struct adb_request *req, int sync); -static int pmu_autopoll(int devs); -void pmu_poll(void); -static int pmu_reset_bus(void); - -static int init_pmu(void); -static void pmu_start(void); -static void send_byte(int x); -static void recv_byte(void); -static void pmu_done(struct adb_request *req); -static void pmu_handle_data(unsigned char *data, int len); -static void set_volume(int level); -static void pmu_enable_backlight(int on); -static void pmu_set_brightness(int level); - -struct adb_driver via_pmu_driver = { - .name = "68K PMU", - .probe = pmu_probe, - .init = pmu_init, - .send_request = pmu_send_request, - .autopoll = pmu_autopoll, - .poll = pmu_poll, - .reset_bus = pmu_reset_bus, -}; - -/* - * This table indicates for each PMU opcode: - * - the number of data bytes to be sent with the command, or -1 - * if a length byte should be sent, - * - the number of response bytes which the PMU will return, or - * -1 if it will send a length byte. - */ -static s8 pmu_data_len[256][2] = { -/* 0 1 2 3 4 5 6 7 */ -/*00*/ {-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*08*/ {-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1}, -/*10*/ { 1, 0},{ 1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*18*/ { 0, 1},{ 0, 1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{ 0, 0}, -/*20*/ {-1, 0},{ 0, 0},{ 2, 0},{ 1, 0},{ 1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*28*/ { 0,-1},{ 0,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{ 0,-1}, -/*30*/ { 4, 0},{20, 0},{-1, 0},{ 3, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*38*/ { 0, 4},{ 0,20},{ 2,-1},{ 2, 1},{ 3,-1},{-1,-1},{-1,-1},{ 4, 0}, -/*40*/ { 1, 0},{ 1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*48*/ { 0, 1},{ 0, 1},{-1,-1},{ 1, 0},{ 1, 0},{-1,-1},{-1,-1},{-1,-1}, -/*50*/ { 1, 0},{ 0, 0},{ 2, 0},{ 2, 0},{-1, 0},{ 1, 0},{ 3, 0},{ 1, 0}, -/*58*/ { 0, 1},{ 1, 0},{ 0, 2},{ 0, 2},{ 0,-1},{-1,-1},{-1,-1},{-1,-1}, -/*60*/ { 2, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*68*/ { 0, 3},{ 0, 3},{ 0, 2},{ 0, 8},{ 0,-1},{ 0,-1},{-1,-1},{-1,-1}, -/*70*/ { 1, 0},{ 1, 0},{ 1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*78*/ { 0,-1},{ 0,-1},{-1,-1},{-1,-1},{-1,-1},{ 5, 1},{ 4, 1},{ 4, 1}, -/*80*/ { 4, 0},{-1, 0},{ 0, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*88*/ { 0, 5},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1}, -/*90*/ { 1, 0},{ 2, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*98*/ { 0, 1},{ 0, 1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1}, -/*a0*/ { 2, 0},{ 2, 0},{ 2, 0},{ 4, 0},{-1, 0},{ 0, 0},{-1, 0},{-1, 0}, -/*a8*/ { 1, 1},{ 1, 0},{ 3, 0},{ 2, 0},{-1,-1},{-1,-1},{-1,-1},{-1,-1}, -/*b0*/ {-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*b8*/ {-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1}, -/*c0*/ {-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*c8*/ {-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1}, -/*d0*/ { 0, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*d8*/ { 1, 1},{ 1, 1},{-1,-1},{-1,-1},{ 0, 1},{ 0,-1},{-1,-1},{-1,-1}, -/*e0*/ {-1, 0},{ 4, 0},{ 0, 1},{-1, 0},{-1, 0},{ 4, 0},{-1, 0},{-1, 0}, -/*e8*/ { 3,-1},{-1,-1},{ 0, 1},{-1,-1},{ 0,-1},{-1,-1},{-1,-1},{ 0, 0}, -/*f0*/ {-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0},{-1, 0}, -/*f8*/ {-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1},{-1,-1}, -}; - -int __init find_via_pmu(void) -{ - switch (macintosh_config->adb_type) { - case MAC_ADB_PB1: - pmu_kind = PMU_68K_V1; - break; - case MAC_ADB_PB2: - pmu_kind = PMU_68K_V2; - break; - default: - pmu_kind = PMU_UNKNOWN; - return -ENODEV; - } - - pmu_state = idle; - - if (!init_pmu()) - goto fail_init; - - pr_info("adb: PMU 68K driver v0.5 for Unified ADB\n"); - - return 1; - -fail_init: - pmu_kind = PMU_UNKNOWN; - return 0; -} - -static int pmu_probe(void) -{ - if (pmu_kind == PMU_UNKNOWN) - return -ENODEV; - return 0; -} - -static int pmu_init(void) -{ - if (pmu_kind == PMU_UNKNOWN) - return -ENODEV; - return 0; -} - -static int __init via_pmu_start(void) -{ - if (pmu_kind == PMU_UNKNOWN) - return -ENODEV; - - if (request_irq(IRQ_MAC_ADB_SR, pmu_interrupt, 0, "PMU_SR", - pmu_interrupt)) { - pr_err("%s: can't get SR irq\n", __func__); - return -ENODEV; - } - if (request_irq(IRQ_MAC_ADB_CL, pmu_interrupt, 0, "PMU_CL", - pmu_interrupt)) { - pr_err("%s: can't get CL irq\n", __func__); - free_irq(IRQ_MAC_ADB_SR, pmu_interrupt); - return -ENODEV; - } - - pmu_fully_inited = 1; - - /* Enable backlight */ - pmu_enable_backlight(1); - - return 0; -} - -arch_initcall(via_pmu_start); - -static int __init init_pmu(void) -{ - int timeout; - volatile struct adb_request req; - - via2[B] |= TREQ; /* negate TREQ */ - via2[DIRB] = (via2[DIRB] | TREQ) & ~TACK; /* TACK in, TREQ out */ - - pmu_request((struct adb_request *) &req, NULL, 2, PMU_SET_INTR_MASK, PMU_INT_ADB); - timeout = 100000; - while (!req.complete) { - if (--timeout < 0) { - printk(KERN_ERR "pmu_init: no response from PMU\n"); - return -EAGAIN; - } - udelay(10); - pmu_poll(); - } - - /* ack all pending interrupts */ - timeout = 100000; - interrupt_data[0] = 1; - while (interrupt_data[0] || pmu_state != idle) { - if (--timeout < 0) { - printk(KERN_ERR "pmu_init: timed out acking intrs\n"); - return -EAGAIN; - } - if (pmu_state == idle) { - adb_int_pending = 1; - pmu_interrupt(0, NULL); - } - pmu_poll(); - udelay(10); - } - - pmu_request((struct adb_request *) &req, NULL, 2, PMU_SET_INTR_MASK, - PMU_INT_ADB_AUTO|PMU_INT_SNDBRT|PMU_INT_ADB); - timeout = 100000; - while (!req.complete) { - if (--timeout < 0) { - printk(KERN_ERR "pmu_init: no response from PMU\n"); - return -EAGAIN; - } - udelay(10); - pmu_poll(); - } - - bright_req_1.complete = 1; - bright_req_2.complete = 1; - bright_req_3.complete = 1; - - return 1; -} - -int -pmu_get_model(void) -{ - return pmu_kind; -} - -/* Send an ADB command */ -static int -pmu_send_request(struct adb_request *req, int sync) -{ - int i, ret; - - if (!pmu_fully_inited) - { - req->complete = 1; - return -ENXIO; - } - - ret = -EINVAL; - - switch (req->data[0]) { - case PMU_PACKET: - for (i = 0; i < req->nbytes - 1; ++i) - req->data[i] = req->data[i+1]; - --req->nbytes; - if (pmu_data_len[req->data[0]][1] != 0) { - req->reply[0] = ADB_RET_OK; - req->reply_len = 1; - } else - req->reply_len = 0; - ret = pmu_queue_request(req); - break; - case CUDA_PACKET: - switch (req->data[1]) { - case CUDA_GET_TIME: - if (req->nbytes != 2) - break; - req->data[0] = PMU_READ_RTC; - req->nbytes = 1; - req->reply_len = 3; - req->reply[0] = CUDA_PACKET; - req->reply[1] = 0; - req->reply[2] = CUDA_GET_TIME; - ret = pmu_queue_request(req); - break; - case CUDA_SET_TIME: - if (req->nbytes != 6) - break; - req->data[0] = PMU_SET_RTC; - req->nbytes = 5; - for (i = 1; i <= 4; ++i) - req->data[i] = req->data[i+1]; - req->reply_len = 3; - req->reply[0] = CUDA_PACKET; - req->reply[1] = 0; - req->reply[2] = CUDA_SET_TIME; - ret = pmu_queue_request(req); - break; - case CUDA_GET_PRAM: - if (req->nbytes != 4) - break; - req->data[0] = PMU_READ_NVRAM; - req->data[1] = req->data[2]; - req->data[2] = req->data[3]; - req->nbytes = 3; - req->reply_len = 3; - req->reply[0] = CUDA_PACKET; - req->reply[1] = 0; - req->reply[2] = CUDA_GET_PRAM; - ret = pmu_queue_request(req); - break; - case CUDA_SET_PRAM: - if (req->nbytes != 5) - break; - req->data[0] = PMU_WRITE_NVRAM; - req->data[1] = req->data[2]; - req->data[2] = req->data[3]; - req->data[3] = req->data[4]; - req->nbytes = 4; - req->reply_len = 3; - req->reply[0] = CUDA_PACKET; - req->reply[1] = 0; - req->reply[2] = CUDA_SET_PRAM; - ret = pmu_queue_request(req); - break; - } - break; - case ADB_PACKET: - for (i = req->nbytes - 1; i > 1; --i) - req->data[i+2] = req->data[i]; - req->data[3] = req->nbytes - 2; - req->data[2] = pmu_adb_flags; - /*req->data[1] = req->data[1];*/ - req->data[0] = PMU_ADB_CMD; - req->nbytes += 2; - req->reply_expected = 1; - req->reply_len = 0; - ret = pmu_queue_request(req); - break; - } - if (ret) - { - req->complete = 1; - return ret; - } - - if (sync) { - while (!req->complete) - pmu_poll(); - } - - return 0; -} - -/* Enable/disable autopolling */ -static int -pmu_autopoll(int devs) -{ - struct adb_request req; - - if (!pmu_fully_inited) return -ENXIO; - - if (devs) { - adb_dev_map = devs; - pmu_request(&req, NULL, 5, PMU_ADB_CMD, 0, 0x86, - adb_dev_map >> 8, adb_dev_map); - pmu_adb_flags = 2; - } else { - pmu_request(&req, NULL, 1, PMU_ADB_POLL_OFF); - pmu_adb_flags = 0; - } - while (!req.complete) - pmu_poll(); - return 0; -} - -/* Reset the ADB bus */ -static int -pmu_reset_bus(void) -{ - struct adb_request req; - long timeout; - int save_autopoll = adb_dev_map; - - if (!pmu_fully_inited) return -ENXIO; - - /* anyone got a better idea?? */ - pmu_autopoll(0); - - req.nbytes = 5; - req.done = NULL; - req.data[0] = PMU_ADB_CMD; - req.data[1] = 0; - req.data[2] = 3; /* ADB_BUSRESET ??? */ - req.data[3] = 0; - req.data[4] = 0; - req.reply_len = 0; - req.reply_expected = 1; - if (pmu_queue_request(&req) != 0) - { - printk(KERN_ERR "pmu_adb_reset_bus: pmu_queue_request failed\n"); - return -EIO; - } - while (!req.complete) - pmu_poll(); - timeout = 100000; - while (!req.complete) { - if (--timeout < 0) { - printk(KERN_ERR "pmu_adb_reset_bus (reset): no response from PMU\n"); - return -EIO; - } - udelay(10); - pmu_poll(); - } - - if (save_autopoll != 0) - pmu_autopoll(save_autopoll); - - return 0; -} - -/* Construct and send a pmu request */ -int -pmu_request(struct adb_request *req, void (*done)(struct adb_request *), - int nbytes, ...) -{ - va_list list; - int i; - - if (nbytes < 0 || nbytes > 32) { - printk(KERN_ERR "pmu_request: bad nbytes (%d)\n", nbytes); - req->complete = 1; - return -EINVAL; - } - req->nbytes = nbytes; - req->done = done; - va_start(list, nbytes); - for (i = 0; i < nbytes; ++i) - req->data[i] = va_arg(list, int); - va_end(list); - if (pmu_data_len[req->data[0]][1] != 0) { - req->reply[0] = ADB_RET_OK; - req->reply_len = 1; - } else - req->reply_len = 0; - req->reply_expected = 0; - return pmu_queue_request(req); -} - -int -pmu_queue_request(struct adb_request *req) -{ - unsigned long flags; - int nsend; - - if (req->nbytes <= 0) { - req->complete = 1; - return 0; - } - nsend = pmu_data_len[req->data[0]][0]; - if (nsend >= 0 && req->nbytes != nsend + 1) { - req->complete = 1; - return -EINVAL; - } - - req->next = NULL; - req->sent = 0; - req->complete = 0; - local_irq_save(flags); - - if (current_req != 0) { - last_req->next = req; - last_req = req; - } else { - current_req = req; - last_req = req; - if (pmu_state == idle) - pmu_start(); - } - - local_irq_restore(flags); - return 0; -} - -static void -send_byte(int x) -{ - via1[ACR] |= SR_CTRL; - via1[SR] = x; - via2[B] &= ~TREQ; /* assert TREQ */ -} - -static void -recv_byte(void) -{ - char c; - - via1[ACR] = (via1[ACR] | SR_EXT) & ~SR_OUT; - c = via1[SR]; /* resets SR */ - via2[B] &= ~TREQ; -} - -static void -pmu_start(void) -{ - unsigned long flags; - struct adb_request *req; - - /* assert pmu_state == idle */ - /* get the packet to send */ - local_irq_save(flags); - req = current_req; - if (req == 0 || pmu_state != idle - || (req->reply_expected && req_awaiting_reply)) - goto out; - - pmu_state = sending; - data_index = 1; - data_len = pmu_data_len[req->data[0]][0]; - - /* set the shift register to shift out and send a byte */ - send_byte(req->data[0]); - -out: - local_irq_restore(flags); -} - -void -pmu_poll(void) -{ - unsigned long flags; - - local_irq_save(flags); - if (via1[IFR] & SR_INT) { - via1[IFR] = SR_INT; - pmu_interrupt(IRQ_MAC_ADB_SR, NULL); - } - if (via1[IFR] & CB1_INT) { - via1[IFR] = CB1_INT; - pmu_interrupt(IRQ_MAC_ADB_CL, NULL); - } - local_irq_restore(flags); -} - -static irqreturn_t -pmu_interrupt(int irq, void *dev_id) -{ - struct adb_request *req; - int timeout, bite = 0; /* to prevent compiler warning */ - -#if 0 - printk("pmu_interrupt: irq %d state %d acr %02X, b %02X data_index %d/%d adb_int_pending %d\n", - irq, pmu_state, (uint) via1[ACR], (uint) via2[B], data_index, data_len, adb_int_pending); -#endif - - if (irq == IRQ_MAC_ADB_CL) { /* CB1 interrupt */ - adb_int_pending = 1; - } else if (irq == IRQ_MAC_ADB_SR) { /* SR interrupt */ - if (via2[B] & TACK) { - printk(KERN_DEBUG "PMU: SR_INT but ack still high! (%x)\n", via2[B]); - } - - /* if reading grab the byte */ - if ((via1[ACR] & SR_OUT) == 0) bite = via1[SR]; - - /* reset TREQ and wait for TACK to go high */ - via2[B] |= TREQ; - timeout = 3200; - while (!(via2[B] & TACK)) { - if (--timeout < 0) { - printk(KERN_ERR "PMU not responding (!ack)\n"); - goto finish; - } - udelay(10); - } - - switch (pmu_state) { - case sending: - req = current_req; - if (data_len < 0) { - data_len = req->nbytes - 1; - send_byte(data_len); - break; - } - if (data_index <= data_len) { - send_byte(req->data[data_index++]); - break; - } - req->sent = 1; - data_len = pmu_data_len[req->data[0]][1]; - if (data_len == 0) { - pmu_state = idle; - current_req = req->next; - if (req->reply_expected) - req_awaiting_reply = req; - else - pmu_done(req); - } else { - pmu_state = reading; - data_index = 0; - reply_ptr = req->reply + req->reply_len; - recv_byte(); - } - break; - - case intack: - data_index = 0; - data_len = -1; - pmu_state = reading_intr; - reply_ptr = interrupt_data; - recv_byte(); - break; - - case reading: - case reading_intr: - if (data_len == -1) { - data_len = bite; - if (bite > 32) - printk(KERN_ERR "PMU: bad reply len %d\n", - bite); - } else { - reply_ptr[data_index++] = bite; - } - if (data_index < data_len) { - recv_byte(); - break; - } - - if (pmu_state == reading_intr) { - pmu_handle_data(interrupt_data, data_index); - } else { - req = current_req; - current_req = req->next; - req->reply_len += data_index; - pmu_done(req); - } - pmu_state = idle; - - break; - - default: - printk(KERN_ERR "pmu_interrupt: unknown state %d?\n", - pmu_state); - } - } -finish: - if (pmu_state == idle) { - if (adb_int_pending) { - pmu_state = intack; - send_byte(PMU_INT_ACK); - adb_int_pending = 0; - } else if (current_req) { - pmu_start(); - } - } - -#if 0 - printk("pmu_interrupt: exit state %d acr %02X, b %02X data_index %d/%d adb_int_pending %d\n", - pmu_state, (uint) via1[ACR], (uint) via2[B], data_index, data_len, adb_int_pending); -#endif - return IRQ_HANDLED; -} - -static void -pmu_done(struct adb_request *req) -{ - req->complete = 1; - if (req->done) - (*req->done)(req); -} - -/* Interrupt data could be the result data from an ADB cmd */ -static void -pmu_handle_data(unsigned char *data, int len) -{ - static int show_pmu_ints = 1; - - asleep = 0; - if (len < 1) { - adb_int_pending = 0; - return; - } - if (data[0] & PMU_INT_ADB) { - if ((data[0] & PMU_INT_ADB_AUTO) == 0) { - struct adb_request *req = req_awaiting_reply; - if (req == 0) { - printk(KERN_ERR "PMU: extra ADB reply\n"); - return; - } - req_awaiting_reply = NULL; - if (len <= 2) - req->reply_len = 0; - else { - memcpy(req->reply, data + 1, len - 1); - req->reply_len = len - 1; - } - pmu_done(req); - } else { - adb_input(data+1, len-1, 1); - } - } else { - if (data[0] == 0x08 && len == 3) { - /* sound/brightness buttons pressed */ - pmu_set_brightness(data[1] >> 3); - set_volume(data[2]); - } else if (show_pmu_ints - && !(data[0] == PMU_INT_TICK && len == 1)) { - int i; - printk(KERN_DEBUG "pmu intr"); - for (i = 0; i < len; ++i) - printk(" %.2x", data[i]); - printk("\n"); - } - } -} - -static int backlight_level = -1; -static int backlight_enabled = 0; - -#define LEVEL_TO_BRIGHT(lev) ((lev) < 1? 0x7f: 0x4a - ((lev) << 1)) - -static void -pmu_enable_backlight(int on) -{ - struct adb_request req; - - if (on) { - /* first call: get current backlight value */ - if (backlight_level < 0) { - switch(pmu_kind) { - case PMU_68K_V1: - case PMU_68K_V2: - pmu_request(&req, NULL, 3, PMU_READ_NVRAM, 0x14, 0xe); - while (!req.complete) - pmu_poll(); - printk(KERN_DEBUG "pmu: nvram returned bright: %d\n", (int)req.reply[1]); - backlight_level = req.reply[1]; - break; - default: - backlight_enabled = 0; - return; - } - } - pmu_request(&req, NULL, 2, PMU_BACKLIGHT_BRIGHT, - LEVEL_TO_BRIGHT(backlight_level)); - while (!req.complete) - pmu_poll(); - } - pmu_request(&req, NULL, 2, PMU_POWER_CTRL, - PMU_POW_BACKLIGHT | (on ? PMU_POW_ON : PMU_POW_OFF)); - while (!req.complete) - pmu_poll(); - backlight_enabled = on; -} - -static void -pmu_set_brightness(int level) -{ - int bright; - - backlight_level = level; - bright = LEVEL_TO_BRIGHT(level); - if (!backlight_enabled) - return; - if (bright_req_1.complete) - pmu_request(&bright_req_1, NULL, 2, PMU_BACKLIGHT_BRIGHT, - bright); - if (bright_req_2.complete) - pmu_request(&bright_req_2, NULL, 2, PMU_POWER_CTRL, - PMU_POW_BACKLIGHT | (bright < 0x7f ? PMU_POW_ON : PMU_POW_OFF)); -} - -void -pmu_enable_irled(int on) -{ - struct adb_request req; - - pmu_request(&req, NULL, 2, PMU_POWER_CTRL, PMU_POW_IRLED | - (on ? PMU_POW_ON : PMU_POW_OFF)); - while (!req.complete) - pmu_poll(); -} - -static void -set_volume(int level) -{ -} - -int -pmu_present(void) -{ - return (pmu_kind != PMU_UNKNOWN); -} diff --git a/drivers/md/dm-cache-metadata.c b/drivers/md/dm-cache-metadata.c index 0d7212410e21..69dddeab124c 100644 --- a/drivers/md/dm-cache-metadata.c +++ b/drivers/md/dm-cache-metadata.c @@ -363,7 +363,7 @@ static int __write_initial_superblock(struct dm_cache_metadata *cmd) disk_super->version = cpu_to_le32(cmd->version); memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name)); memset(disk_super->policy_version, 0, sizeof(disk_super->policy_version)); - disk_super->policy_hint_size = 0; + disk_super->policy_hint_size = cpu_to_le32(0); __copy_sm_root(cmd, disk_super); @@ -701,6 +701,7 @@ static int __commit_transaction(struct dm_cache_metadata *cmd, disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]); disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]); disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]); + disk_super->policy_hint_size = cpu_to_le32(cmd->policy_hint_size); disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits); disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses); @@ -1322,6 +1323,7 @@ static int __load_mapping_v1(struct dm_cache_metadata *cmd, dm_oblock_t oblock; unsigned flags; + bool dirty = true; dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le); memcpy(&mapping, mapping_value_le, sizeof(mapping)); @@ -1332,8 +1334,10 @@ static int __load_mapping_v1(struct dm_cache_metadata *cmd, dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le); memcpy(&hint, hint_value_le, sizeof(hint)); } + if (cmd->clean_when_opened) + dirty = flags & M_DIRTY; - r = fn(context, oblock, to_cblock(cb), flags & M_DIRTY, + r = fn(context, oblock, to_cblock(cb), dirty, le32_to_cpu(hint), hints_valid); if (r) { DMERR("policy couldn't load cache block %llu", @@ -1361,7 +1365,7 @@ static int __load_mapping_v2(struct dm_cache_metadata *cmd, dm_oblock_t oblock; unsigned flags; - bool dirty; + bool dirty = true; dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le); memcpy(&mapping, mapping_value_le, sizeof(mapping)); @@ -1372,8 +1376,9 @@ static int __load_mapping_v2(struct dm_cache_metadata *cmd, dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le); memcpy(&hint, hint_value_le, sizeof(hint)); } + if (cmd->clean_when_opened) + dirty = dm_bitset_cursor_get_value(dirty_cursor); - dirty = dm_bitset_cursor_get_value(dirty_cursor); r = fn(context, oblock, to_cblock(cb), dirty, le32_to_cpu(hint), hints_valid); if (r) { diff --git a/drivers/md/dm-cache-target.c b/drivers/md/dm-cache-target.c index ce14a3d1f609..a53413371725 100644 --- a/drivers/md/dm-cache-target.c +++ b/drivers/md/dm-cache-target.c @@ -1188,9 +1188,8 @@ static void copy_complete(int read_err, unsigned long write_err, void *context) queue_continuation(mg->cache->wq, &mg->k); } -static int copy(struct dm_cache_migration *mg, bool promote) +static void copy(struct dm_cache_migration *mg, bool promote) { - int r; struct dm_io_region o_region, c_region; struct cache *cache = mg->cache; @@ -1203,11 +1202,9 @@ static int copy(struct dm_cache_migration *mg, bool promote) c_region.count = cache->sectors_per_block; if (promote) - r = dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k); + dm_kcopyd_copy(cache->copier, &o_region, 1, &c_region, 0, copy_complete, &mg->k); else - r = dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k); - - return r; + dm_kcopyd_copy(cache->copier, &c_region, 1, &o_region, 0, copy_complete, &mg->k); } static void bio_drop_shared_lock(struct cache *cache, struct bio *bio) @@ -1449,12 +1446,7 @@ static void mg_full_copy(struct work_struct *ws) } init_continuation(&mg->k, mg_upgrade_lock); - - if (copy(mg, is_policy_promote)) { - DMERR_LIMIT("%s: migration copy failed", cache_device_name(cache)); - mg->k.input = BLK_STS_IOERR; - mg_complete(mg, false); - } + copy(mg, is_policy_promote); } static void mg_copy(struct work_struct *ws) @@ -2250,7 +2242,7 @@ static int parse_features(struct cache_args *ca, struct dm_arg_set *as, {0, 2, "Invalid number of cache feature arguments"}, }; - int r; + int r, mode_ctr = 0; unsigned argc; const char *arg; struct cache_features *cf = &ca->features; @@ -2264,14 +2256,20 @@ static int parse_features(struct cache_args *ca, struct dm_arg_set *as, while (argc--) { arg = dm_shift_arg(as); - if (!strcasecmp(arg, "writeback")) + if (!strcasecmp(arg, "writeback")) { cf->io_mode = CM_IO_WRITEBACK; + mode_ctr++; + } - else if (!strcasecmp(arg, "writethrough")) + else if (!strcasecmp(arg, "writethrough")) { cf->io_mode = CM_IO_WRITETHROUGH; + mode_ctr++; + } - else if (!strcasecmp(arg, "passthrough")) + else if (!strcasecmp(arg, "passthrough")) { cf->io_mode = CM_IO_PASSTHROUGH; + mode_ctr++; + } else if (!strcasecmp(arg, "metadata2")) cf->metadata_version = 2; @@ -2282,6 +2280,11 @@ static int parse_features(struct cache_args *ca, struct dm_arg_set *as, } } + if (mode_ctr > 1) { + *error = "Duplicate cache io_mode features requested"; + return -EINVAL; + } + return 0; } diff --git a/drivers/md/dm-crypt.c b/drivers/md/dm-crypt.c index b61b069c33af..f266c81f396f 100644 --- a/drivers/md/dm-crypt.c +++ b/drivers/md/dm-crypt.c @@ -99,7 +99,7 @@ struct crypt_iv_operations { }; struct iv_essiv_private { - struct crypto_ahash *hash_tfm; + struct crypto_shash *hash_tfm; u8 *salt; }; @@ -144,7 +144,7 @@ struct crypt_config { struct workqueue_struct *io_queue; struct workqueue_struct *crypt_queue; - wait_queue_head_t write_thread_wait; + spinlock_t write_thread_lock; struct task_struct *write_thread; struct rb_root write_tree; @@ -327,25 +327,22 @@ static int crypt_iv_plain64be_gen(struct crypt_config *cc, u8 *iv, static int crypt_iv_essiv_init(struct crypt_config *cc) { struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; - AHASH_REQUEST_ON_STACK(req, essiv->hash_tfm); - struct scatterlist sg; + SHASH_DESC_ON_STACK(desc, essiv->hash_tfm); struct crypto_cipher *essiv_tfm; int err; - sg_init_one(&sg, cc->key, cc->key_size); - ahash_request_set_tfm(req, essiv->hash_tfm); - ahash_request_set_callback(req, CRYPTO_TFM_REQ_MAY_SLEEP, NULL, NULL); - ahash_request_set_crypt(req, &sg, essiv->salt, cc->key_size); + desc->tfm = essiv->hash_tfm; + desc->flags = CRYPTO_TFM_REQ_MAY_SLEEP; - err = crypto_ahash_digest(req); - ahash_request_zero(req); + err = crypto_shash_digest(desc, cc->key, cc->key_size, essiv->salt); + shash_desc_zero(desc); if (err) return err; essiv_tfm = cc->iv_private; err = crypto_cipher_setkey(essiv_tfm, essiv->salt, - crypto_ahash_digestsize(essiv->hash_tfm)); + crypto_shash_digestsize(essiv->hash_tfm)); if (err) return err; @@ -356,7 +353,7 @@ static int crypt_iv_essiv_init(struct crypt_config *cc) static int crypt_iv_essiv_wipe(struct crypt_config *cc) { struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; - unsigned salt_size = crypto_ahash_digestsize(essiv->hash_tfm); + unsigned salt_size = crypto_shash_digestsize(essiv->hash_tfm); struct crypto_cipher *essiv_tfm; int r, err = 0; @@ -408,7 +405,7 @@ static void crypt_iv_essiv_dtr(struct crypt_config *cc) struct crypto_cipher *essiv_tfm; struct iv_essiv_private *essiv = &cc->iv_gen_private.essiv; - crypto_free_ahash(essiv->hash_tfm); + crypto_free_shash(essiv->hash_tfm); essiv->hash_tfm = NULL; kzfree(essiv->salt); @@ -426,7 +423,7 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, const char *opts) { struct crypto_cipher *essiv_tfm = NULL; - struct crypto_ahash *hash_tfm = NULL; + struct crypto_shash *hash_tfm = NULL; u8 *salt = NULL; int err; @@ -436,14 +433,14 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, } /* Allocate hash algorithm */ - hash_tfm = crypto_alloc_ahash(opts, 0, CRYPTO_ALG_ASYNC); + hash_tfm = crypto_alloc_shash(opts, 0, 0); if (IS_ERR(hash_tfm)) { ti->error = "Error initializing ESSIV hash"; err = PTR_ERR(hash_tfm); goto bad; } - salt = kzalloc(crypto_ahash_digestsize(hash_tfm), GFP_KERNEL); + salt = kzalloc(crypto_shash_digestsize(hash_tfm), GFP_KERNEL); if (!salt) { ti->error = "Error kmallocing salt storage in ESSIV"; err = -ENOMEM; @@ -454,7 +451,7 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, cc->iv_gen_private.essiv.hash_tfm = hash_tfm; essiv_tfm = alloc_essiv_cipher(cc, ti, salt, - crypto_ahash_digestsize(hash_tfm)); + crypto_shash_digestsize(hash_tfm)); if (IS_ERR(essiv_tfm)) { crypt_iv_essiv_dtr(cc); return PTR_ERR(essiv_tfm); @@ -465,7 +462,7 @@ static int crypt_iv_essiv_ctr(struct crypt_config *cc, struct dm_target *ti, bad: if (hash_tfm && !IS_ERR(hash_tfm)) - crypto_free_ahash(hash_tfm); + crypto_free_shash(hash_tfm); kfree(salt); return err; } @@ -1620,36 +1617,31 @@ static int dmcrypt_write(void *data) struct rb_root write_tree; struct blk_plug plug; - DECLARE_WAITQUEUE(wait, current); - - spin_lock_irq(&cc->write_thread_wait.lock); + spin_lock_irq(&cc->write_thread_lock); continue_locked: if (!RB_EMPTY_ROOT(&cc->write_tree)) goto pop_from_list; set_current_state(TASK_INTERRUPTIBLE); - __add_wait_queue(&cc->write_thread_wait, &wait); - spin_unlock_irq(&cc->write_thread_wait.lock); + spin_unlock_irq(&cc->write_thread_lock); if (unlikely(kthread_should_stop())) { set_current_state(TASK_RUNNING); - remove_wait_queue(&cc->write_thread_wait, &wait); break; } schedule(); set_current_state(TASK_RUNNING); - spin_lock_irq(&cc->write_thread_wait.lock); - __remove_wait_queue(&cc->write_thread_wait, &wait); + spin_lock_irq(&cc->write_thread_lock); goto continue_locked; pop_from_list: write_tree = cc->write_tree; cc->write_tree = RB_ROOT; - spin_unlock_irq(&cc->write_thread_wait.lock); + spin_unlock_irq(&cc->write_thread_lock); BUG_ON(rb_parent(write_tree.rb_node)); @@ -1693,7 +1685,9 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) return; } - spin_lock_irqsave(&cc->write_thread_wait.lock, flags); + spin_lock_irqsave(&cc->write_thread_lock, flags); + if (RB_EMPTY_ROOT(&cc->write_tree)) + wake_up_process(cc->write_thread); rbp = &cc->write_tree.rb_node; parent = NULL; sector = io->sector; @@ -1706,9 +1700,7 @@ static void kcryptd_crypt_write_io_submit(struct dm_crypt_io *io, int async) } rb_link_node(&io->rb_node, parent, rbp); rb_insert_color(&io->rb_node, &cc->write_tree); - - wake_up_locked(&cc->write_thread_wait); - spin_unlock_irqrestore(&cc->write_thread_wait.lock, flags); + spin_unlock_irqrestore(&cc->write_thread_lock, flags); } static void kcryptd_crypt_write_convert(struct dm_crypt_io *io) @@ -2831,7 +2823,7 @@ static int crypt_ctr(struct dm_target *ti, unsigned int argc, char **argv) goto bad; } - init_waitqueue_head(&cc->write_thread_wait); + spin_lock_init(&cc->write_thread_lock); cc->write_tree = RB_ROOT; cc->write_thread = kthread_create(dmcrypt_write, cc, "dmcrypt_write"); @@ -3069,11 +3061,11 @@ static void crypt_io_hints(struct dm_target *ti, struct queue_limits *limits) */ limits->max_segment_size = PAGE_SIZE; - if (cc->sector_size != (1 << SECTOR_SHIFT)) { - limits->logical_block_size = cc->sector_size; - limits->physical_block_size = cc->sector_size; - blk_limits_io_min(limits, cc->sector_size); - } + limits->logical_block_size = + max_t(unsigned short, limits->logical_block_size, cc->sector_size); + limits->physical_block_size = + max_t(unsigned, limits->physical_block_size, cc->sector_size); + limits->io_min = max_t(unsigned, limits->io_min, cc->sector_size); } static struct target_type crypt_target = { diff --git a/drivers/md/dm-delay.c b/drivers/md/dm-delay.c index 1783d80c9cad..2fb7bb4304ad 100644 --- a/drivers/md/dm-delay.c +++ b/drivers/md/dm-delay.c @@ -17,6 +17,13 @@ #define DM_MSG_PREFIX "delay" +struct delay_class { + struct dm_dev *dev; + sector_t start; + unsigned delay; + unsigned ops; +}; + struct delay_c { struct timer_list delay_timer; struct mutex timer_lock; @@ -25,19 +32,16 @@ struct delay_c { struct list_head delayed_bios; atomic_t may_delay; - struct dm_dev *dev_read; - sector_t start_read; - unsigned read_delay; - unsigned reads; + struct delay_class read; + struct delay_class write; + struct delay_class flush; - struct dm_dev *dev_write; - sector_t start_write; - unsigned write_delay; - unsigned writes; + int argc; }; struct dm_delay_info { struct delay_c *context; + struct delay_class *class; struct list_head list; unsigned long expires; }; @@ -77,7 +81,7 @@ static struct bio *flush_delayed_bios(struct delay_c *dc, int flush_all) { struct dm_delay_info *delayed, *next; unsigned long next_expires = 0; - int start_timer = 0; + unsigned long start_timer = 0; struct bio_list flush_bios = { }; mutex_lock(&delayed_bios_lock); @@ -87,10 +91,7 @@ static struct bio *flush_delayed_bios(struct delay_c *dc, int flush_all) sizeof(struct dm_delay_info)); list_del(&delayed->list); bio_list_add(&flush_bios, bio); - if ((bio_data_dir(bio) == WRITE)) - delayed->context->writes--; - else - delayed->context->reads--; + delayed->class->ops--; continue; } @@ -100,7 +101,6 @@ static struct bio *flush_delayed_bios(struct delay_c *dc, int flush_all) } else next_expires = min(next_expires, delayed->expires); } - mutex_unlock(&delayed_bios_lock); if (start_timer) @@ -117,6 +117,50 @@ static void flush_expired_bios(struct work_struct *work) flush_bios(flush_delayed_bios(dc, 0)); } +static void delay_dtr(struct dm_target *ti) +{ + struct delay_c *dc = ti->private; + + destroy_workqueue(dc->kdelayd_wq); + + if (dc->read.dev) + dm_put_device(ti, dc->read.dev); + if (dc->write.dev) + dm_put_device(ti, dc->write.dev); + if (dc->flush.dev) + dm_put_device(ti, dc->flush.dev); + + mutex_destroy(&dc->timer_lock); + + kfree(dc); +} + +static int delay_class_ctr(struct dm_target *ti, struct delay_class *c, char **argv) +{ + int ret; + unsigned long long tmpll; + char dummy; + + if (sscanf(argv[1], "%llu%c", &tmpll, &dummy) != 1) { + ti->error = "Invalid device sector"; + return -EINVAL; + } + c->start = tmpll; + + if (sscanf(argv[2], "%u%c", &c->delay, &dummy) != 1) { + ti->error = "Invalid delay"; + return -EINVAL; + } + + ret = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &c->dev); + if (ret) { + ti->error = "Device lookup failed"; + return ret; + } + + return 0; +} + /* * Mapping parameters: * <device> <offset> <delay> [<write_device> <write_offset> <write_delay>] @@ -128,134 +172,89 @@ static void flush_expired_bios(struct work_struct *work) static int delay_ctr(struct dm_target *ti, unsigned int argc, char **argv) { struct delay_c *dc; - unsigned long long tmpll; - char dummy; int ret; - if (argc != 3 && argc != 6) { - ti->error = "Requires exactly 3 or 6 arguments"; + if (argc != 3 && argc != 6 && argc != 9) { + ti->error = "Requires exactly 3, 6 or 9 arguments"; return -EINVAL; } - dc = kmalloc(sizeof(*dc), GFP_KERNEL); + dc = kzalloc(sizeof(*dc), GFP_KERNEL); if (!dc) { ti->error = "Cannot allocate context"; return -ENOMEM; } - dc->reads = dc->writes = 0; + ti->private = dc; + timer_setup(&dc->delay_timer, handle_delayed_timer, 0); + INIT_WORK(&dc->flush_expired_bios, flush_expired_bios); + INIT_LIST_HEAD(&dc->delayed_bios); + mutex_init(&dc->timer_lock); + atomic_set(&dc->may_delay, 1); + dc->argc = argc; - ret = -EINVAL; - if (sscanf(argv[1], "%llu%c", &tmpll, &dummy) != 1) { - ti->error = "Invalid device sector"; + ret = delay_class_ctr(ti, &dc->read, argv); + if (ret) goto bad; - } - dc->start_read = tmpll; - if (sscanf(argv[2], "%u%c", &dc->read_delay, &dummy) != 1) { - ti->error = "Invalid delay"; - goto bad; + if (argc == 3) { + ret = delay_class_ctr(ti, &dc->write, argv); + if (ret) + goto bad; + ret = delay_class_ctr(ti, &dc->flush, argv); + if (ret) + goto bad; + goto out; } - ret = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), - &dc->dev_read); - if (ret) { - ti->error = "Device lookup failed"; + ret = delay_class_ctr(ti, &dc->write, argv + 3); + if (ret) goto bad; - } - - ret = -EINVAL; - dc->dev_write = NULL; - if (argc == 3) + if (argc == 6) { + ret = delay_class_ctr(ti, &dc->flush, argv + 3); + if (ret) + goto bad; goto out; - - if (sscanf(argv[4], "%llu%c", &tmpll, &dummy) != 1) { - ti->error = "Invalid write device sector"; - goto bad_dev_read; } - dc->start_write = tmpll; - if (sscanf(argv[5], "%u%c", &dc->write_delay, &dummy) != 1) { - ti->error = "Invalid write delay"; - goto bad_dev_read; - } - - ret = dm_get_device(ti, argv[3], dm_table_get_mode(ti->table), - &dc->dev_write); - if (ret) { - ti->error = "Write device lookup failed"; - goto bad_dev_read; - } + ret = delay_class_ctr(ti, &dc->flush, argv + 6); + if (ret) + goto bad; out: - ret = -EINVAL; dc->kdelayd_wq = alloc_workqueue("kdelayd", WQ_MEM_RECLAIM, 0); if (!dc->kdelayd_wq) { + ret = -EINVAL; DMERR("Couldn't start kdelayd"); - goto bad_queue; + goto bad; } - timer_setup(&dc->delay_timer, handle_delayed_timer, 0); - - INIT_WORK(&dc->flush_expired_bios, flush_expired_bios); - INIT_LIST_HEAD(&dc->delayed_bios); - mutex_init(&dc->timer_lock); - atomic_set(&dc->may_delay, 1); - ti->num_flush_bios = 1; ti->num_discard_bios = 1; ti->per_io_data_size = sizeof(struct dm_delay_info); - ti->private = dc; return 0; -bad_queue: - if (dc->dev_write) - dm_put_device(ti, dc->dev_write); -bad_dev_read: - dm_put_device(ti, dc->dev_read); bad: - kfree(dc); + delay_dtr(ti); return ret; } -static void delay_dtr(struct dm_target *ti) -{ - struct delay_c *dc = ti->private; - - destroy_workqueue(dc->kdelayd_wq); - - dm_put_device(ti, dc->dev_read); - - if (dc->dev_write) - dm_put_device(ti, dc->dev_write); - - mutex_destroy(&dc->timer_lock); - - kfree(dc); -} - -static int delay_bio(struct delay_c *dc, int delay, struct bio *bio) +static int delay_bio(struct delay_c *dc, struct delay_class *c, struct bio *bio) { struct dm_delay_info *delayed; unsigned long expires = 0; - if (!delay || !atomic_read(&dc->may_delay)) + if (!c->delay || !atomic_read(&dc->may_delay)) return DM_MAPIO_REMAPPED; delayed = dm_per_bio_data(bio, sizeof(struct dm_delay_info)); delayed->context = dc; - delayed->expires = expires = jiffies + msecs_to_jiffies(delay); + delayed->expires = expires = jiffies + msecs_to_jiffies(c->delay); mutex_lock(&delayed_bios_lock); - - if (bio_data_dir(bio) == WRITE) - dc->writes++; - else - dc->reads++; - + c->ops++; list_add_tail(&delayed->list, &dc->delayed_bios); - mutex_unlock(&delayed_bios_lock); queue_timeout(dc, expires); @@ -282,23 +281,28 @@ static void delay_resume(struct dm_target *ti) static int delay_map(struct dm_target *ti, struct bio *bio) { struct delay_c *dc = ti->private; - - if ((bio_data_dir(bio) == WRITE) && (dc->dev_write)) { - bio_set_dev(bio, dc->dev_write->bdev); - if (bio_sectors(bio)) - bio->bi_iter.bi_sector = dc->start_write + - dm_target_offset(ti, bio->bi_iter.bi_sector); - - return delay_bio(dc, dc->write_delay, bio); + struct delay_class *c; + struct dm_delay_info *delayed = dm_per_bio_data(bio, sizeof(struct dm_delay_info)); + + if (bio_data_dir(bio) == WRITE) { + if (unlikely(bio->bi_opf & REQ_PREFLUSH)) + c = &dc->flush; + else + c = &dc->write; + } else { + c = &dc->read; } + delayed->class = c; + bio_set_dev(bio, c->dev->bdev); + if (bio_sectors(bio)) + bio->bi_iter.bi_sector = c->start + dm_target_offset(ti, bio->bi_iter.bi_sector); - bio_set_dev(bio, dc->dev_read->bdev); - bio->bi_iter.bi_sector = dc->start_read + - dm_target_offset(ti, bio->bi_iter.bi_sector); - - return delay_bio(dc, dc->read_delay, bio); + return delay_bio(dc, c, bio); } +#define DMEMIT_DELAY_CLASS(c) \ + DMEMIT("%s %llu %u", (c)->dev->name, (unsigned long long)(c)->start, (c)->delay) + static void delay_status(struct dm_target *ti, status_type_t type, unsigned status_flags, char *result, unsigned maxlen) { @@ -307,17 +311,19 @@ static void delay_status(struct dm_target *ti, status_type_t type, switch (type) { case STATUSTYPE_INFO: - DMEMIT("%u %u", dc->reads, dc->writes); + DMEMIT("%u %u %u", dc->read.ops, dc->write.ops, dc->flush.ops); break; case STATUSTYPE_TABLE: - DMEMIT("%s %llu %u", dc->dev_read->name, - (unsigned long long) dc->start_read, - dc->read_delay); - if (dc->dev_write) - DMEMIT(" %s %llu %u", dc->dev_write->name, - (unsigned long long) dc->start_write, - dc->write_delay); + DMEMIT_DELAY_CLASS(&dc->read); + if (dc->argc >= 6) { + DMEMIT(" "); + DMEMIT_DELAY_CLASS(&dc->write); + } + if (dc->argc >= 9) { + DMEMIT(" "); + DMEMIT_DELAY_CLASS(&dc->flush); + } break; } } @@ -328,12 +334,15 @@ static int delay_iterate_devices(struct dm_target *ti, struct delay_c *dc = ti->private; int ret = 0; - ret = fn(ti, dc->dev_read, dc->start_read, ti->len, data); + ret = fn(ti, dc->read.dev, dc->read.start, ti->len, data); + if (ret) + goto out; + ret = fn(ti, dc->write.dev, dc->write.start, ti->len, data); + if (ret) + goto out; + ret = fn(ti, dc->flush.dev, dc->flush.start, ti->len, data); if (ret) goto out; - - if (dc->dev_write) - ret = fn(ti, dc->dev_write, dc->start_write, ti->len, data); out: return ret; diff --git a/drivers/md/dm-integrity.c b/drivers/md/dm-integrity.c index 86438b2f10dd..378878599466 100644 --- a/drivers/md/dm-integrity.c +++ b/drivers/md/dm-integrity.c @@ -31,6 +31,8 @@ #define MIN_LOG2_INTERLEAVE_SECTORS 3 #define MAX_LOG2_INTERLEAVE_SECTORS 31 #define METADATA_WORKQUEUE_MAX_ACTIVE 16 +#define RECALC_SECTORS 8192 +#define RECALC_WRITE_SUPER 16 /* * Warning - DEBUG_PRINT prints security-sensitive data to the log, @@ -44,7 +46,8 @@ */ #define SB_MAGIC "integrt" -#define SB_VERSION 1 +#define SB_VERSION_1 1 +#define SB_VERSION_2 2 #define SB_SECTORS 8 #define MAX_SECTORS_PER_BLOCK 8 @@ -57,9 +60,12 @@ struct superblock { __u64 provided_data_sectors; /* userspace uses this value */ __u32 flags; __u8 log2_sectors_per_block; + __u8 pad[3]; + __u64 recalc_sector; }; #define SB_FLAG_HAVE_JOURNAL_MAC 0x1 +#define SB_FLAG_RECALCULATING 0x2 #define JOURNAL_ENTRY_ROUNDUP 8 @@ -139,6 +145,7 @@ struct alg_spec { struct dm_integrity_c { struct dm_dev *dev; + struct dm_dev *meta_dev; unsigned tag_size; __s8 log2_tag_size; sector_t start; @@ -170,7 +177,8 @@ struct dm_integrity_c { unsigned short journal_section_sectors; unsigned journal_sections; unsigned journal_entries; - sector_t device_sectors; + sector_t data_device_sectors; + sector_t meta_device_sectors; unsigned initial_sectors; unsigned metadata_run; __s8 log2_metadata_run; @@ -178,7 +186,7 @@ struct dm_integrity_c { __u8 sectors_per_block; unsigned char mode; - bool suspending; + int suspending; int failed; @@ -186,6 +194,7 @@ struct dm_integrity_c { /* these variables are locked with endio_wait.lock */ struct rb_root in_progress; + struct list_head wait_list; wait_queue_head_t endio_wait; struct workqueue_struct *wait_wq; @@ -210,6 +219,11 @@ struct dm_integrity_c { struct workqueue_struct *writer_wq; struct work_struct writer_work; + struct workqueue_struct *recalc_wq; + struct work_struct recalc_work; + u8 *recalc_buffer; + u8 *recalc_tags; + struct bio_list flush_bio_list; unsigned long autocommit_jiffies; @@ -233,7 +247,14 @@ struct dm_integrity_c { struct dm_integrity_range { sector_t logical_sector; unsigned n_sectors; - struct rb_node node; + bool waiting; + union { + struct rb_node node; + struct { + struct task_struct *task; + struct list_head wait_entry; + }; + }; }; struct dm_integrity_io { @@ -337,10 +358,14 @@ static commit_id_t dm_integrity_commit_id(struct dm_integrity_c *ic, unsigned i, static void get_area_and_offset(struct dm_integrity_c *ic, sector_t data_sector, sector_t *area, sector_t *offset) { - __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors; - - *area = data_sector >> log2_interleave_sectors; - *offset = (unsigned)data_sector & ((1U << log2_interleave_sectors) - 1); + if (!ic->meta_dev) { + __u8 log2_interleave_sectors = ic->sb->log2_interleave_sectors; + *area = data_sector >> log2_interleave_sectors; + *offset = (unsigned)data_sector & ((1U << log2_interleave_sectors) - 1); + } else { + *area = 0; + *offset = data_sector; + } } #define sector_to_block(ic, n) \ @@ -379,6 +404,9 @@ static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector { sector_t result; + if (ic->meta_dev) + return offset; + result = area << ic->sb->log2_interleave_sectors; if (likely(ic->log2_metadata_run >= 0)) result += (area + 1) << ic->log2_metadata_run; @@ -386,6 +414,8 @@ static sector_t get_data_sector(struct dm_integrity_c *ic, sector_t area, sector result += (area + 1) * ic->metadata_run; result += (sector_t)ic->initial_sectors + offset; + result += ic->start; + return result; } @@ -395,6 +425,14 @@ static void wraparound_section(struct dm_integrity_c *ic, unsigned *sec_ptr) *sec_ptr -= ic->journal_sections; } +static void sb_set_version(struct dm_integrity_c *ic) +{ + if (ic->meta_dev || ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) + ic->sb->version = SB_VERSION_2; + else + ic->sb->version = SB_VERSION_1; +} + static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags) { struct dm_io_request io_req; @@ -406,7 +444,7 @@ static int sync_rw_sb(struct dm_integrity_c *ic, int op, int op_flags) io_req.mem.ptr.addr = ic->sb; io_req.notify.fn = NULL; io_req.client = ic->io; - io_loc.bdev = ic->dev->bdev; + io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; io_loc.sector = ic->start; io_loc.count = SB_SECTORS; @@ -753,7 +791,7 @@ static void rw_journal(struct dm_integrity_c *ic, int op, int op_flags, unsigned io_req.notify.fn = NULL; } io_req.client = ic->io; - io_loc.bdev = ic->dev->bdev; + io_loc.bdev = ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev; io_loc.sector = ic->start + SB_SECTORS + sector; io_loc.count = n_sectors; @@ -857,7 +895,7 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsig io_req.notify.context = data; io_req.client = ic->io; io_loc.bdev = ic->dev->bdev; - io_loc.sector = ic->start + target; + io_loc.sector = target; io_loc.count = n_sectors; r = dm_io(&io_req, 1, &io_loc, NULL); @@ -867,13 +905,27 @@ static void copy_from_journal(struct dm_integrity_c *ic, unsigned section, unsig } } -static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) +static bool ranges_overlap(struct dm_integrity_range *range1, struct dm_integrity_range *range2) +{ + return range1->logical_sector < range2->logical_sector + range2->n_sectors && + range2->logical_sector + range2->n_sectors > range2->logical_sector; +} + +static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range, bool check_waiting) { struct rb_node **n = &ic->in_progress.rb_node; struct rb_node *parent; BUG_ON((new_range->logical_sector | new_range->n_sectors) & (unsigned)(ic->sectors_per_block - 1)); + if (likely(check_waiting)) { + struct dm_integrity_range *range; + list_for_each_entry(range, &ic->wait_list, wait_entry) { + if (unlikely(ranges_overlap(range, new_range))) + return false; + } + } + parent = NULL; while (*n) { @@ -898,7 +950,22 @@ static bool add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range * static void remove_range_unlocked(struct dm_integrity_c *ic, struct dm_integrity_range *range) { rb_erase(&range->node, &ic->in_progress); - wake_up_locked(&ic->endio_wait); + while (unlikely(!list_empty(&ic->wait_list))) { + struct dm_integrity_range *last_range = + list_first_entry(&ic->wait_list, struct dm_integrity_range, wait_entry); + struct task_struct *last_range_task; + if (!ranges_overlap(range, last_range)) + break; + last_range_task = last_range->task; + list_del(&last_range->wait_entry); + if (!add_new_range(ic, last_range, false)) { + last_range->task = last_range_task; + list_add(&last_range->wait_entry, &ic->wait_list); + break; + } + last_range->waiting = false; + wake_up_process(last_range_task); + } } static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *range) @@ -910,6 +977,19 @@ static void remove_range(struct dm_integrity_c *ic, struct dm_integrity_range *r spin_unlock_irqrestore(&ic->endio_wait.lock, flags); } +static void wait_and_add_new_range(struct dm_integrity_c *ic, struct dm_integrity_range *new_range) +{ + new_range->waiting = true; + list_add_tail(&new_range->wait_entry, &ic->wait_list); + new_range->task = current; + do { + __set_current_state(TASK_UNINTERRUPTIBLE); + spin_unlock_irq(&ic->endio_wait.lock); + io_schedule(); + spin_lock_irq(&ic->endio_wait.lock); + } while (unlikely(new_range->waiting)); +} + static void init_journal_node(struct journal_node *node) { RB_CLEAR_NODE(&node->node); @@ -1599,8 +1679,12 @@ retry: dio->range.n_sectors = min(dio->range.n_sectors, ic->free_sectors << ic->sb->log2_sectors_per_block); - if (unlikely(!dio->range.n_sectors)) - goto sleep; + if (unlikely(!dio->range.n_sectors)) { + if (from_map) + goto offload_to_thread; + sleep_on_endio_wait(ic); + goto retry; + } range_sectors = dio->range.n_sectors >> ic->sb->log2_sectors_per_block; ic->free_sectors -= range_sectors; journal_section = ic->free_section; @@ -1654,22 +1738,20 @@ retry: } } } - if (unlikely(!add_new_range(ic, &dio->range))) { + if (unlikely(!add_new_range(ic, &dio->range, true))) { /* * We must not sleep in the request routine because it could * stall bios on current->bio_list. * So, we offload the bio to a workqueue if we have to sleep. */ -sleep: if (from_map) { +offload_to_thread: spin_unlock_irq(&ic->endio_wait.lock); INIT_WORK(&dio->work, integrity_bio_wait); queue_work(ic->wait_wq, &dio->work); return; - } else { - sleep_on_endio_wait(ic); - goto retry; } + wait_and_add_new_range(ic, &dio->range); } spin_unlock_irq(&ic->endio_wait.lock); @@ -1701,14 +1783,18 @@ sleep: bio->bi_end_io = integrity_end_io; bio->bi_iter.bi_size = dio->range.n_sectors << SECTOR_SHIFT; - bio->bi_iter.bi_sector += ic->start; generic_make_request(bio); if (need_sync_io) { wait_for_completion_io(&read_comp); + if (unlikely(ic->recalc_wq != NULL) && + ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING) && + dio->range.logical_sector + dio->range.n_sectors > le64_to_cpu(ic->sb->recalc_sector)) + goto skip_check; if (likely(!bio->bi_status)) integrity_metadata(&dio->work); else +skip_check: dec_in_flight(dio); } else { @@ -1892,8 +1978,8 @@ static void do_journal_write(struct dm_integrity_c *ic, unsigned write_start, io->range.n_sectors = (k - j) << ic->sb->log2_sectors_per_block; spin_lock_irq(&ic->endio_wait.lock); - while (unlikely(!add_new_range(ic, &io->range))) - sleep_on_endio_wait(ic); + if (unlikely(!add_new_range(ic, &io->range, true))) + wait_and_add_new_range(ic, &io->range); if (likely(!from_replay)) { struct journal_node *section_node = &ic->journal_tree[i * ic->journal_section_entries]; @@ -1981,7 +2067,7 @@ static void integrity_writer(struct work_struct *w) unsigned prev_free_sectors; /* the following test is not needed, but it tests the replay code */ - if (READ_ONCE(ic->suspending)) + if (READ_ONCE(ic->suspending) && !ic->meta_dev) return; spin_lock_irq(&ic->endio_wait.lock); @@ -2008,6 +2094,108 @@ static void integrity_writer(struct work_struct *w) spin_unlock_irq(&ic->endio_wait.lock); } +static void recalc_write_super(struct dm_integrity_c *ic) +{ + int r; + + dm_integrity_flush_buffers(ic); + if (dm_integrity_failed(ic)) + return; + + sb_set_version(ic); + r = sync_rw_sb(ic, REQ_OP_WRITE, 0); + if (unlikely(r)) + dm_integrity_io_error(ic, "writing superblock", r); +} + +static void integrity_recalc(struct work_struct *w) +{ + struct dm_integrity_c *ic = container_of(w, struct dm_integrity_c, recalc_work); + struct dm_integrity_range range; + struct dm_io_request io_req; + struct dm_io_region io_loc; + sector_t area, offset; + sector_t metadata_block; + unsigned metadata_offset; + __u8 *t; + unsigned i; + int r; + unsigned super_counter = 0; + + spin_lock_irq(&ic->endio_wait.lock); + +next_chunk: + + if (unlikely(READ_ONCE(ic->suspending))) + goto unlock_ret; + + range.logical_sector = le64_to_cpu(ic->sb->recalc_sector); + if (unlikely(range.logical_sector >= ic->provided_data_sectors)) + goto unlock_ret; + + get_area_and_offset(ic, range.logical_sector, &area, &offset); + range.n_sectors = min((sector_t)RECALC_SECTORS, ic->provided_data_sectors - range.logical_sector); + if (!ic->meta_dev) + range.n_sectors = min(range.n_sectors, (1U << ic->sb->log2_interleave_sectors) - (unsigned)offset); + + if (unlikely(!add_new_range(ic, &range, true))) + wait_and_add_new_range(ic, &range); + + spin_unlock_irq(&ic->endio_wait.lock); + + if (unlikely(++super_counter == RECALC_WRITE_SUPER)) { + recalc_write_super(ic); + super_counter = 0; + } + + if (unlikely(dm_integrity_failed(ic))) + goto err; + + io_req.bi_op = REQ_OP_READ; + io_req.bi_op_flags = 0; + io_req.mem.type = DM_IO_VMA; + io_req.mem.ptr.addr = ic->recalc_buffer; + io_req.notify.fn = NULL; + io_req.client = ic->io; + io_loc.bdev = ic->dev->bdev; + io_loc.sector = get_data_sector(ic, area, offset); + io_loc.count = range.n_sectors; + + r = dm_io(&io_req, 1, &io_loc, NULL); + if (unlikely(r)) { + dm_integrity_io_error(ic, "reading data", r); + goto err; + } + + t = ic->recalc_tags; + for (i = 0; i < range.n_sectors; i += ic->sectors_per_block) { + integrity_sector_checksum(ic, range.logical_sector + i, ic->recalc_buffer + (i << SECTOR_SHIFT), t); + t += ic->tag_size; + } + + metadata_block = get_metadata_sector_and_offset(ic, area, offset, &metadata_offset); + + r = dm_integrity_rw_tag(ic, ic->recalc_tags, &metadata_block, &metadata_offset, t - ic->recalc_tags, TAG_WRITE); + if (unlikely(r)) { + dm_integrity_io_error(ic, "writing tags", r); + goto err; + } + + spin_lock_irq(&ic->endio_wait.lock); + remove_range_unlocked(ic, &range); + ic->sb->recalc_sector = cpu_to_le64(range.logical_sector + range.n_sectors); + goto next_chunk; + +err: + remove_range(ic, &range); + return; + +unlock_ret: + spin_unlock_irq(&ic->endio_wait.lock); + + recalc_write_super(ic); +} + static void init_journal(struct dm_integrity_c *ic, unsigned start_section, unsigned n_sections, unsigned char commit_seq) { @@ -2210,17 +2398,22 @@ static void dm_integrity_postsuspend(struct dm_target *ti) del_timer_sync(&ic->autocommit_timer); - ic->suspending = true; + WRITE_ONCE(ic->suspending, 1); + + if (ic->recalc_wq) + drain_workqueue(ic->recalc_wq); queue_work(ic->commit_wq, &ic->commit_work); drain_workqueue(ic->commit_wq); if (ic->mode == 'J') { + if (ic->meta_dev) + queue_work(ic->writer_wq, &ic->writer_work); drain_workqueue(ic->writer_wq); dm_integrity_flush_buffers(ic); } - ic->suspending = false; + WRITE_ONCE(ic->suspending, 0); BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); @@ -2232,6 +2425,16 @@ static void dm_integrity_resume(struct dm_target *ti) struct dm_integrity_c *ic = (struct dm_integrity_c *)ti->private; replay_journal(ic); + + if (ic->recalc_wq && ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { + __u64 recalc_pos = le64_to_cpu(ic->sb->recalc_sector); + if (recalc_pos < ic->provided_data_sectors) { + queue_work(ic->recalc_wq, &ic->recalc_work); + } else if (recalc_pos > ic->provided_data_sectors) { + ic->sb->recalc_sector = cpu_to_le64(ic->provided_data_sectors); + recalc_write_super(ic); + } + } } static void dm_integrity_status(struct dm_target *ti, status_type_t type, @@ -2243,7 +2446,13 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, switch (type) { case STATUSTYPE_INFO: - DMEMIT("%llu", (unsigned long long)atomic64_read(&ic->number_of_mismatches)); + DMEMIT("%llu %llu", + (unsigned long long)atomic64_read(&ic->number_of_mismatches), + (unsigned long long)ic->provided_data_sectors); + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) + DMEMIT(" %llu", (unsigned long long)le64_to_cpu(ic->sb->recalc_sector)); + else + DMEMIT(" -"); break; case STATUSTYPE_TABLE: { @@ -2251,19 +2460,25 @@ static void dm_integrity_status(struct dm_target *ti, status_type_t type, watermark_percentage += ic->journal_entries / 2; do_div(watermark_percentage, ic->journal_entries); arg_count = 5; + arg_count += !!ic->meta_dev; arg_count += ic->sectors_per_block != 1; + arg_count += !!(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)); arg_count += !!ic->internal_hash_alg.alg_string; arg_count += !!ic->journal_crypt_alg.alg_string; arg_count += !!ic->journal_mac_alg.alg_string; DMEMIT("%s %llu %u %c %u", ic->dev->name, (unsigned long long)ic->start, ic->tag_size, ic->mode, arg_count); + if (ic->meta_dev) + DMEMIT(" meta_device:%s", ic->meta_dev->name); + if (ic->sectors_per_block != 1) + DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT); + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) + DMEMIT(" recalculate"); DMEMIT(" journal_sectors:%u", ic->initial_sectors - SB_SECTORS); DMEMIT(" interleave_sectors:%u", 1U << ic->sb->log2_interleave_sectors); DMEMIT(" buffer_sectors:%u", 1U << ic->log2_buffer_sectors); DMEMIT(" journal_watermark:%u", (unsigned)watermark_percentage); DMEMIT(" commit_time:%u", ic->autocommit_msec); - if (ic->sectors_per_block != 1) - DMEMIT(" block_size:%u", ic->sectors_per_block << SECTOR_SHIFT); #define EMIT_ALG(a, n) \ do { \ @@ -2286,7 +2501,10 @@ static int dm_integrity_iterate_devices(struct dm_target *ti, { struct dm_integrity_c *ic = ti->private; - return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data); + if (!ic->meta_dev) + return fn(ti, ic->dev, ic->start + ic->initial_sectors + ic->metadata_run, ti->len, data); + else + return fn(ti, ic->dev, 0, ti->len, data); } static void dm_integrity_io_hints(struct dm_target *ti, struct queue_limits *limits) @@ -2319,26 +2537,38 @@ static void calculate_journal_section_size(struct dm_integrity_c *ic) static int calculate_device_limits(struct dm_integrity_c *ic) { __u64 initial_sectors; - sector_t last_sector, last_area, last_offset; calculate_journal_section_size(ic); initial_sectors = SB_SECTORS + (__u64)ic->journal_section_sectors * ic->journal_sections; - if (initial_sectors + METADATA_PADDING_SECTORS >= ic->device_sectors || initial_sectors > UINT_MAX) + if (initial_sectors + METADATA_PADDING_SECTORS >= ic->meta_device_sectors || initial_sectors > UINT_MAX) return -EINVAL; ic->initial_sectors = initial_sectors; - ic->metadata_run = roundup((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block), - (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS)) >> SECTOR_SHIFT; - if (!(ic->metadata_run & (ic->metadata_run - 1))) - ic->log2_metadata_run = __ffs(ic->metadata_run); - else - ic->log2_metadata_run = -1; + if (!ic->meta_dev) { + sector_t last_sector, last_area, last_offset; - get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset); - last_sector = get_data_sector(ic, last_area, last_offset); + ic->metadata_run = roundup((__u64)ic->tag_size << (ic->sb->log2_interleave_sectors - ic->sb->log2_sectors_per_block), + (__u64)(1 << SECTOR_SHIFT << METADATA_PADDING_SECTORS)) >> SECTOR_SHIFT; + if (!(ic->metadata_run & (ic->metadata_run - 1))) + ic->log2_metadata_run = __ffs(ic->metadata_run); + else + ic->log2_metadata_run = -1; - if (ic->start + last_sector < last_sector || ic->start + last_sector >= ic->device_sectors) - return -EINVAL; + get_area_and_offset(ic, ic->provided_data_sectors - 1, &last_area, &last_offset); + last_sector = get_data_sector(ic, last_area, last_offset); + if (last_sector < ic->start || last_sector >= ic->meta_device_sectors) + return -EINVAL; + } else { + __u64 meta_size = ic->provided_data_sectors * ic->tag_size; + meta_size = (meta_size + ((1U << (ic->log2_buffer_sectors + SECTOR_SHIFT)) - 1)) + >> (ic->log2_buffer_sectors + SECTOR_SHIFT); + meta_size <<= ic->log2_buffer_sectors; + if (ic->initial_sectors + meta_size < ic->initial_sectors || + ic->initial_sectors + meta_size > ic->meta_device_sectors) + return -EINVAL; + ic->metadata_run = 1; + ic->log2_metadata_run = 0; + } return 0; } @@ -2350,7 +2580,6 @@ static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sec memset(ic->sb, 0, SB_SECTORS << SECTOR_SHIFT); memcpy(ic->sb->magic, SB_MAGIC, 8); - ic->sb->version = SB_VERSION; ic->sb->integrity_tag_size = cpu_to_le16(ic->tag_size); ic->sb->log2_sectors_per_block = __ffs(ic->sectors_per_block); if (ic->journal_mac_alg.alg_string) @@ -2360,28 +2589,55 @@ static int initialize_superblock(struct dm_integrity_c *ic, unsigned journal_sec journal_sections = journal_sectors / ic->journal_section_sectors; if (!journal_sections) journal_sections = 1; - ic->sb->journal_sections = cpu_to_le32(journal_sections); - if (!interleave_sectors) - interleave_sectors = DEFAULT_INTERLEAVE_SECTORS; - ic->sb->log2_interleave_sectors = __fls(interleave_sectors); - ic->sb->log2_interleave_sectors = max((__u8)MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); - ic->sb->log2_interleave_sectors = min((__u8)MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); - - ic->provided_data_sectors = 0; - for (test_bit = fls64(ic->device_sectors) - 1; test_bit >= 3; test_bit--) { - __u64 prev_data_sectors = ic->provided_data_sectors; + if (!ic->meta_dev) { + ic->sb->journal_sections = cpu_to_le32(journal_sections); + if (!interleave_sectors) + interleave_sectors = DEFAULT_INTERLEAVE_SECTORS; + ic->sb->log2_interleave_sectors = __fls(interleave_sectors); + ic->sb->log2_interleave_sectors = max((__u8)MIN_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); + ic->sb->log2_interleave_sectors = min((__u8)MAX_LOG2_INTERLEAVE_SECTORS, ic->sb->log2_interleave_sectors); + + ic->provided_data_sectors = 0; + for (test_bit = fls64(ic->meta_device_sectors) - 1; test_bit >= 3; test_bit--) { + __u64 prev_data_sectors = ic->provided_data_sectors; + + ic->provided_data_sectors |= (sector_t)1 << test_bit; + if (calculate_device_limits(ic)) + ic->provided_data_sectors = prev_data_sectors; + } + if (!ic->provided_data_sectors) + return -EINVAL; + } else { + ic->sb->log2_interleave_sectors = 0; + ic->provided_data_sectors = ic->data_device_sectors; + ic->provided_data_sectors &= ~(sector_t)(ic->sectors_per_block - 1); + +try_smaller_buffer: + ic->sb->journal_sections = cpu_to_le32(0); + for (test_bit = fls(journal_sections) - 1; test_bit >= 0; test_bit--) { + __u32 prev_journal_sections = le32_to_cpu(ic->sb->journal_sections); + __u32 test_journal_sections = prev_journal_sections | (1U << test_bit); + if (test_journal_sections > journal_sections) + continue; + ic->sb->journal_sections = cpu_to_le32(test_journal_sections); + if (calculate_device_limits(ic)) + ic->sb->journal_sections = cpu_to_le32(prev_journal_sections); - ic->provided_data_sectors |= (sector_t)1 << test_bit; - if (calculate_device_limits(ic)) - ic->provided_data_sectors = prev_data_sectors; + } + if (!le32_to_cpu(ic->sb->journal_sections)) { + if (ic->log2_buffer_sectors > 3) { + ic->log2_buffer_sectors--; + goto try_smaller_buffer; + } + return -EINVAL; + } } - if (!ic->provided_data_sectors) - return -EINVAL; - ic->sb->provided_data_sectors = cpu_to_le64(ic->provided_data_sectors); + sb_set_version(ic); + return 0; } @@ -2828,6 +3084,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) {0, 9, "Invalid number of feature args"}, }; unsigned journal_sectors, interleave_sectors, buffer_sectors, journal_watermark, sync_msec; + bool recalculate; bool should_write_sb; __u64 threshold; unsigned long long start; @@ -2848,6 +3105,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) ti->per_io_data_size = sizeof(struct dm_integrity_io); ic->in_progress = RB_ROOT; + INIT_LIST_HEAD(&ic->wait_list); init_waitqueue_head(&ic->endio_wait); bio_list_init(&ic->flush_bio_list); init_waitqueue_head(&ic->copy_to_journal_wait); @@ -2883,13 +3141,12 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) goto bad; } - ic->device_sectors = i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT; - journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS, - ic->device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR); + journal_sectors = 0; interleave_sectors = DEFAULT_INTERLEAVE_SECTORS; buffer_sectors = DEFAULT_BUFFER_SECTORS; journal_watermark = DEFAULT_JOURNAL_WATERMARK; sync_msec = DEFAULT_SYNC_MSEC; + recalculate = false; ic->sectors_per_block = 1; as.argc = argc - DIRECT_ARGUMENTS; @@ -2908,7 +3165,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) goto bad; } if (sscanf(opt_string, "journal_sectors:%u%c", &val, &dummy) == 1) - journal_sectors = val; + journal_sectors = val ? val : 1; else if (sscanf(opt_string, "interleave_sectors:%u%c", &val, &dummy) == 1) interleave_sectors = val; else if (sscanf(opt_string, "buffer_sectors:%u%c", &val, &dummy) == 1) @@ -2917,7 +3174,17 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) journal_watermark = val; else if (sscanf(opt_string, "commit_time:%u%c", &val, &dummy) == 1) sync_msec = val; - else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) { + else if (!memcmp(opt_string, "meta_device:", strlen("meta_device:"))) { + if (ic->meta_dev) { + dm_put_device(ti, ic->meta_dev); + ic->meta_dev = NULL; + } + r = dm_get_device(ti, strchr(opt_string, ':') + 1, dm_table_get_mode(ti->table), &ic->meta_dev); + if (r) { + ti->error = "Device lookup failed"; + goto bad; + } + } else if (sscanf(opt_string, "block_size:%u%c", &val, &dummy) == 1) { if (val < 1 << SECTOR_SHIFT || val > MAX_SECTORS_PER_BLOCK << SECTOR_SHIFT || (val & (val -1))) { @@ -2941,6 +3208,8 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) "Invalid journal_mac argument"); if (r) goto bad; + } else if (!strcmp(opt_string, "recalculate")) { + recalculate = true; } else { r = -EINVAL; ti->error = "Invalid argument"; @@ -2948,6 +3217,21 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) } } + ic->data_device_sectors = i_size_read(ic->dev->bdev->bd_inode) >> SECTOR_SHIFT; + if (!ic->meta_dev) + ic->meta_device_sectors = ic->data_device_sectors; + else + ic->meta_device_sectors = i_size_read(ic->meta_dev->bdev->bd_inode) >> SECTOR_SHIFT; + + if (!journal_sectors) { + journal_sectors = min((sector_t)DEFAULT_MAX_JOURNAL_SECTORS, + ic->data_device_sectors >> DEFAULT_JOURNAL_SIZE_FACTOR); + } + + if (!buffer_sectors) + buffer_sectors = 1; + ic->log2_buffer_sectors = min((int)__fls(buffer_sectors), 31 - SECTOR_SHIFT); + r = get_mac(&ic->internal_hash, &ic->internal_hash_alg, &ti->error, "Invalid internal hash", "Error setting internal hash key"); if (r) @@ -3062,7 +3346,7 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) should_write_sb = true; } - if (ic->sb->version != SB_VERSION) { + if (!ic->sb->version || ic->sb->version > SB_VERSION_2) { r = -EINVAL; ti->error = "Unknown version"; goto bad; @@ -3083,11 +3367,19 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) goto bad; } /* make sure that ti->max_io_len doesn't overflow */ - if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS || - ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) { - r = -EINVAL; - ti->error = "Invalid interleave_sectors in the superblock"; - goto bad; + if (!ic->meta_dev) { + if (ic->sb->log2_interleave_sectors < MIN_LOG2_INTERLEAVE_SECTORS || + ic->sb->log2_interleave_sectors > MAX_LOG2_INTERLEAVE_SECTORS) { + r = -EINVAL; + ti->error = "Invalid interleave_sectors in the superblock"; + goto bad; + } + } else { + if (ic->sb->log2_interleave_sectors) { + r = -EINVAL; + ti->error = "Invalid interleave_sectors in the superblock"; + goto bad; + } } ic->provided_data_sectors = le64_to_cpu(ic->sb->provided_data_sectors); if (ic->provided_data_sectors != le64_to_cpu(ic->sb->provided_data_sectors)) { @@ -3101,20 +3393,28 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) ti->error = "Journal mac mismatch"; goto bad; } + +try_smaller_buffer: r = calculate_device_limits(ic); if (r) { + if (ic->meta_dev) { + if (ic->log2_buffer_sectors > 3) { + ic->log2_buffer_sectors--; + goto try_smaller_buffer; + } + } ti->error = "The device is too small"; goto bad; } + if (!ic->meta_dev) + ic->log2_buffer_sectors = min(ic->log2_buffer_sectors, (__u8)__ffs(ic->metadata_run)); + if (ti->len > ic->provided_data_sectors) { r = -EINVAL; ti->error = "Not enough provided sectors for requested mapping size"; goto bad; } - if (!buffer_sectors) - buffer_sectors = 1; - ic->log2_buffer_sectors = min3((int)__fls(buffer_sectors), (int)__ffs(ic->metadata_run), 31 - SECTOR_SHIFT); threshold = (__u64)ic->journal_entries * (100 - journal_watermark); threshold += 50; @@ -3138,8 +3438,40 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) (unsigned long long)ic->provided_data_sectors); DEBUG_print(" log2_buffer_sectors %u\n", ic->log2_buffer_sectors); - ic->bufio = dm_bufio_client_create(ic->dev->bdev, 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), - 1, 0, NULL, NULL); + if (recalculate && !(ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING))) { + ic->sb->flags |= cpu_to_le32(SB_FLAG_RECALCULATING); + ic->sb->recalc_sector = cpu_to_le64(0); + } + + if (ic->sb->flags & cpu_to_le32(SB_FLAG_RECALCULATING)) { + if (!ic->internal_hash) { + r = -EINVAL; + ti->error = "Recalculate is only valid with internal hash"; + goto bad; + } + ic->recalc_wq = alloc_workqueue("dm-intergrity-recalc", WQ_MEM_RECLAIM, 1); + if (!ic->recalc_wq ) { + ti->error = "Cannot allocate workqueue"; + r = -ENOMEM; + goto bad; + } + INIT_WORK(&ic->recalc_work, integrity_recalc); + ic->recalc_buffer = vmalloc(RECALC_SECTORS << SECTOR_SHIFT); + if (!ic->recalc_buffer) { + ti->error = "Cannot allocate buffer for recalculating"; + r = -ENOMEM; + goto bad; + } + ic->recalc_tags = kvmalloc((RECALC_SECTORS >> ic->sb->log2_sectors_per_block) * ic->tag_size, GFP_KERNEL); + if (!ic->recalc_tags) { + ti->error = "Cannot allocate tags for recalculating"; + r = -ENOMEM; + goto bad; + } + } + + ic->bufio = dm_bufio_client_create(ic->meta_dev ? ic->meta_dev->bdev : ic->dev->bdev, + 1U << (SECTOR_SHIFT + ic->log2_buffer_sectors), 1, 0, NULL, NULL); if (IS_ERR(ic->bufio)) { r = PTR_ERR(ic->bufio); ti->error = "Cannot initialize dm-bufio"; @@ -3171,9 +3503,11 @@ static int dm_integrity_ctr(struct dm_target *ti, unsigned argc, char **argv) ic->just_formatted = true; } - r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors); - if (r) - goto bad; + if (!ic->meta_dev) { + r = dm_set_target_max_io_len(ti, 1U << ic->sb->log2_interleave_sectors); + if (r) + goto bad; + } if (!ic->internal_hash) dm_integrity_set(ti, ic); @@ -3192,6 +3526,7 @@ static void dm_integrity_dtr(struct dm_target *ti) struct dm_integrity_c *ic = ti->private; BUG_ON(!RB_EMPTY_ROOT(&ic->in_progress)); + BUG_ON(!list_empty(&ic->wait_list)); if (ic->metadata_wq) destroy_workqueue(ic->metadata_wq); @@ -3201,6 +3536,12 @@ static void dm_integrity_dtr(struct dm_target *ti) destroy_workqueue(ic->commit_wq); if (ic->writer_wq) destroy_workqueue(ic->writer_wq); + if (ic->recalc_wq) + destroy_workqueue(ic->recalc_wq); + if (ic->recalc_buffer) + vfree(ic->recalc_buffer); + if (ic->recalc_tags) + kvfree(ic->recalc_tags); if (ic->bufio) dm_bufio_client_destroy(ic->bufio); mempool_exit(&ic->journal_io_mempool); @@ -3208,6 +3549,8 @@ static void dm_integrity_dtr(struct dm_target *ti) dm_io_client_destroy(ic->io); if (ic->dev) dm_put_device(ti, ic->dev); + if (ic->meta_dev) + dm_put_device(ti, ic->meta_dev); dm_integrity_free_page_list(ic, ic->journal); dm_integrity_free_page_list(ic, ic->journal_io); dm_integrity_free_page_list(ic, ic->journal_xor); @@ -3248,7 +3591,7 @@ static void dm_integrity_dtr(struct dm_target *ti) static struct target_type integrity_target = { .name = "integrity", - .version = {1, 1, 0}, + .version = {1, 2, 0}, .module = THIS_MODULE, .features = DM_TARGET_SINGLETON | DM_TARGET_INTEGRITY, .ctr = dm_integrity_ctr, diff --git a/drivers/md/dm-kcopyd.c b/drivers/md/dm-kcopyd.c index 3c7547a3c371..2fc4213e02b5 100644 --- a/drivers/md/dm-kcopyd.c +++ b/drivers/md/dm-kcopyd.c @@ -487,6 +487,8 @@ static int run_complete_job(struct kcopyd_job *job) if (atomic_dec_and_test(&kc->nr_jobs)) wake_up(&kc->destroyq); + cond_resched(); + return 0; } @@ -741,9 +743,9 @@ static void split_job(struct kcopyd_job *master_job) } } -int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, - unsigned int num_dests, struct dm_io_region *dests, - unsigned int flags, dm_kcopyd_notify_fn fn, void *context) +void dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, + unsigned int num_dests, struct dm_io_region *dests, + unsigned int flags, dm_kcopyd_notify_fn fn, void *context) { struct kcopyd_job *job; int i; @@ -818,16 +820,14 @@ int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, job->progress = 0; split_job(job); } - - return 0; } EXPORT_SYMBOL(dm_kcopyd_copy); -int dm_kcopyd_zero(struct dm_kcopyd_client *kc, - unsigned num_dests, struct dm_io_region *dests, - unsigned flags, dm_kcopyd_notify_fn fn, void *context) +void dm_kcopyd_zero(struct dm_kcopyd_client *kc, + unsigned num_dests, struct dm_io_region *dests, + unsigned flags, dm_kcopyd_notify_fn fn, void *context) { - return dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context); + dm_kcopyd_copy(kc, NULL, num_dests, dests, flags, fn, context); } EXPORT_SYMBOL(dm_kcopyd_zero); diff --git a/drivers/md/dm-raid1.c b/drivers/md/dm-raid1.c index 5903e492bb34..79eab1071ec2 100644 --- a/drivers/md/dm-raid1.c +++ b/drivers/md/dm-raid1.c @@ -326,9 +326,8 @@ static void recovery_complete(int read_err, unsigned long write_err, dm_rh_recovery_end(reg, !(read_err || write_err)); } -static int recover(struct mirror_set *ms, struct dm_region *reg) +static void recover(struct mirror_set *ms, struct dm_region *reg) { - int r; unsigned i; struct dm_io_region from, to[DM_KCOPYD_MAX_REGIONS], *dest; struct mirror *m; @@ -367,10 +366,8 @@ static int recover(struct mirror_set *ms, struct dm_region *reg) if (!errors_handled(ms)) set_bit(DM_KCOPYD_IGNORE_ERROR, &flags); - r = dm_kcopyd_copy(ms->kcopyd_client, &from, ms->nr_mirrors - 1, to, - flags, recovery_complete, reg); - - return r; + dm_kcopyd_copy(ms->kcopyd_client, &from, ms->nr_mirrors - 1, to, + flags, recovery_complete, reg); } static void reset_ms_flags(struct mirror_set *ms) @@ -388,7 +385,6 @@ static void do_recovery(struct mirror_set *ms) { struct dm_region *reg; struct dm_dirty_log *log = dm_rh_dirty_log(ms->rh); - int r; /* * Start quiescing some regions. @@ -398,11 +394,8 @@ static void do_recovery(struct mirror_set *ms) /* * Copy any already quiesced regions. */ - while ((reg = dm_rh_recovery_start(ms->rh))) { - r = recover(ms, reg); - if (r) - dm_rh_recovery_end(reg, 0); - } + while ((reg = dm_rh_recovery_start(ms->rh))) + recover(ms, reg); /* * Update the in sync flag. diff --git a/drivers/md/dm-snap.c b/drivers/md/dm-snap.c index 97de7a7334d4..ae4b33d10924 100644 --- a/drivers/md/dm-snap.c +++ b/drivers/md/dm-snap.c @@ -85,7 +85,7 @@ struct dm_snapshot { * A list of pending exceptions that completed out of order. * Protected by kcopyd single-threaded callback. */ - struct list_head out_of_order_list; + struct rb_root out_of_order_tree; mempool_t pending_pool; @@ -200,7 +200,7 @@ struct dm_snap_pending_exception { /* A sequence number, it is used for in-order completion. */ sector_t exception_sequence; - struct list_head out_of_order_entry; + struct rb_node out_of_order_node; /* * For writing a complete chunk, bypassing the copy. @@ -1173,7 +1173,7 @@ static int snapshot_ctr(struct dm_target *ti, unsigned int argc, char **argv) atomic_set(&s->pending_exceptions_count, 0); s->exception_start_sequence = 0; s->exception_complete_sequence = 0; - INIT_LIST_HEAD(&s->out_of_order_list); + s->out_of_order_tree = RB_ROOT; mutex_init(&s->lock); INIT_LIST_HEAD(&s->list); spin_lock_init(&s->pe_lock); @@ -1539,28 +1539,41 @@ static void copy_callback(int read_err, unsigned long write_err, void *context) pe->copy_error = read_err || write_err; if (pe->exception_sequence == s->exception_complete_sequence) { + struct rb_node *next; + s->exception_complete_sequence++; complete_exception(pe); - while (!list_empty(&s->out_of_order_list)) { - pe = list_entry(s->out_of_order_list.next, - struct dm_snap_pending_exception, out_of_order_entry); + next = rb_first(&s->out_of_order_tree); + while (next) { + pe = rb_entry(next, struct dm_snap_pending_exception, + out_of_order_node); if (pe->exception_sequence != s->exception_complete_sequence) break; + next = rb_next(next); s->exception_complete_sequence++; - list_del(&pe->out_of_order_entry); + rb_erase(&pe->out_of_order_node, &s->out_of_order_tree); complete_exception(pe); + cond_resched(); } } else { - struct list_head *lh; + struct rb_node *parent = NULL; + struct rb_node **p = &s->out_of_order_tree.rb_node; struct dm_snap_pending_exception *pe2; - list_for_each_prev(lh, &s->out_of_order_list) { - pe2 = list_entry(lh, struct dm_snap_pending_exception, out_of_order_entry); - if (pe2->exception_sequence < pe->exception_sequence) - break; + while (*p) { + pe2 = rb_entry(*p, struct dm_snap_pending_exception, out_of_order_node); + parent = *p; + + BUG_ON(pe->exception_sequence == pe2->exception_sequence); + if (pe->exception_sequence < pe2->exception_sequence) + p = &((*p)->rb_left); + else + p = &((*p)->rb_right); } - list_add(&pe->out_of_order_entry, lh); + + rb_link_node(&pe->out_of_order_node, parent, p); + rb_insert_color(&pe->out_of_order_node, &s->out_of_order_tree); } } @@ -1694,8 +1707,6 @@ static int snapshot_map(struct dm_target *ti, struct bio *bio) if (!s->valid) return DM_MAPIO_KILL; - /* FIXME: should only take write lock if we need - * to copy an exception */ mutex_lock(&s->lock); if (!s->valid || (unlikely(s->snapshot_overflowed) && diff --git a/drivers/md/dm-thin.c b/drivers/md/dm-thin.c index b900723bbd0f..7bd60a150f8f 100644 --- a/drivers/md/dm-thin.c +++ b/drivers/md/dm-thin.c @@ -1220,18 +1220,13 @@ static struct dm_thin_new_mapping *get_next_mapping(struct pool *pool) static void ll_zero(struct thin_c *tc, struct dm_thin_new_mapping *m, sector_t begin, sector_t end) { - int r; struct dm_io_region to; to.bdev = tc->pool_dev->bdev; to.sector = begin; to.count = end - begin; - r = dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m); - if (r < 0) { - DMERR_LIMIT("dm_kcopyd_zero() failed"); - copy_complete(1, 1, m); - } + dm_kcopyd_zero(tc->pool->copier, 1, &to, 0, copy_complete, m); } static void remap_and_issue_overwrite(struct thin_c *tc, struct bio *bio, @@ -1257,7 +1252,6 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, struct dm_bio_prison_cell *cell, struct bio *bio, sector_t len) { - int r; struct pool *pool = tc->pool; struct dm_thin_new_mapping *m = get_next_mapping(pool); @@ -1296,19 +1290,8 @@ static void schedule_copy(struct thin_c *tc, dm_block_t virt_block, to.sector = data_dest * pool->sectors_per_block; to.count = len; - r = dm_kcopyd_copy(pool->copier, &from, 1, &to, - 0, copy_complete, m); - if (r < 0) { - DMERR_LIMIT("dm_kcopyd_copy() failed"); - copy_complete(1, 1, m); - - /* - * We allow the zero to be issued, to simplify the - * error path. Otherwise we'd need to start - * worrying about decrementing the prepare_actions - * counter. - */ - } + dm_kcopyd_copy(pool->copier, &from, 1, &to, + 0, copy_complete, m); /* * Do we need to zero a tail region? @@ -2520,6 +2503,8 @@ static void set_pool_mode(struct pool *pool, enum pool_mode new_mode) case PM_WRITE: if (old_mode != new_mode) notify_of_pool_mode_change(pool, "write"); + if (old_mode == PM_OUT_OF_DATA_SPACE) + cancel_delayed_work_sync(&pool->no_space_timeout); pool->out_of_data_space = false; pool->pf.error_if_no_space = pt->requested_pf.error_if_no_space; dm_pool_metadata_read_write(pool->pmd); @@ -3890,6 +3875,8 @@ static void pool_status(struct dm_target *ti, status_type_t type, else DMEMIT("- "); + DMEMIT("%llu ", (unsigned long long)calc_metadata_threshold(pt)); + break; case STATUSTYPE_TABLE: @@ -3979,7 +3966,7 @@ static struct target_type pool_target = { .name = "thin-pool", .features = DM_TARGET_SINGLETON | DM_TARGET_ALWAYS_WRITEABLE | DM_TARGET_IMMUTABLE, - .version = {1, 19, 0}, + .version = {1, 20, 0}, .module = THIS_MODULE, .ctr = pool_ctr, .dtr = pool_dtr, @@ -4353,7 +4340,7 @@ static void thin_io_hints(struct dm_target *ti, struct queue_limits *limits) static struct target_type thin_target = { .name = "thin", - .version = {1, 19, 0}, + .version = {1, 20, 0}, .module = THIS_MODULE, .ctr = thin_ctr, .dtr = thin_dtr, diff --git a/drivers/md/dm-writecache.c b/drivers/md/dm-writecache.c index 87107c995cb5..3a28a68f184c 100644 --- a/drivers/md/dm-writecache.c +++ b/drivers/md/dm-writecache.c @@ -457,7 +457,7 @@ static void ssd_commit_flushed(struct dm_writecache *wc) COMPLETION_INITIALIZER_ONSTACK(endio.c), ATOMIC_INIT(1), }; - unsigned bitmap_bits = wc->dirty_bitmap_size * BITS_PER_LONG; + unsigned bitmap_bits = wc->dirty_bitmap_size * 8; unsigned i = 0; while (1) { @@ -2240,6 +2240,8 @@ static void writecache_status(struct dm_target *ti, status_type_t type, DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's', wc->dev->name, wc->ssd_dev->name, wc->block_size); extra_args = 0; + if (wc->start_sector) + extra_args += 2; if (wc->high_wm_percent_set) extra_args += 2; if (wc->low_wm_percent_set) @@ -2254,6 +2256,8 @@ static void writecache_status(struct dm_target *ti, status_type_t type, extra_args++; DMEMIT("%u", extra_args); + if (wc->start_sector) + DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector); if (wc->high_wm_percent_set) { x = (uint64_t)wc->freelist_high_watermark * 100; x += wc->n_blocks / 2; @@ -2280,7 +2284,7 @@ static void writecache_status(struct dm_target *ti, status_type_t type, static struct target_type writecache_target = { .name = "writecache", - .version = {1, 1, 0}, + .version = {1, 1, 1}, .module = THIS_MODULE, .ctr = writecache_ctr, .dtr = writecache_dtr, diff --git a/drivers/md/dm-zoned-reclaim.c b/drivers/md/dm-zoned-reclaim.c index 44a119e12f1a..edf4b95eb075 100644 --- a/drivers/md/dm-zoned-reclaim.c +++ b/drivers/md/dm-zoned-reclaim.c @@ -161,10 +161,8 @@ static int dmz_reclaim_copy(struct dmz_reclaim *zrc, /* Copy the valid region */ set_bit(DMZ_RECLAIM_KCOPY, &zrc->flags); - ret = dm_kcopyd_copy(zrc->kc, &src, 1, &dst, flags, - dmz_reclaim_kcopy_end, zrc); - if (ret) - return ret; + dm_kcopyd_copy(zrc->kc, &src, 1, &dst, flags, + dmz_reclaim_kcopy_end, zrc); /* Wait for copy to complete */ wait_on_bit_io(&zrc->flags, DMZ_RECLAIM_KCOPY, diff --git a/drivers/misc/cxl/Kconfig b/drivers/misc/cxl/Kconfig index 93397cb05b15..3ce933707828 100644 --- a/drivers/misc/cxl/Kconfig +++ b/drivers/misc/cxl/Kconfig @@ -33,11 +33,3 @@ config CXL CAPI adapters are found in POWER8 based systems. If unsure, say N. - -config CXL_BIMODAL - bool "Support for bi-modal CAPI cards" - depends on HOTPLUG_PCI_POWERNV = y && CXL || HOTPLUG_PCI_POWERNV = m && CXL = m - default y - help - Select this option to enable support for bi-modal CAPI cards, such as - the Mellanox CX-4. diff --git a/drivers/misc/cxl/Makefile b/drivers/misc/cxl/Makefile index 502d41fc9ea5..5eea61b9584f 100644 --- a/drivers/misc/cxl/Makefile +++ b/drivers/misc/cxl/Makefile @@ -4,7 +4,7 @@ ccflags-$(CONFIG_PPC_WERROR) += -Werror cxl-y += main.o file.o irq.o fault.o native.o cxl-y += context.o sysfs.o pci.o trace.o -cxl-y += vphb.o phb.o api.o cxllib.o +cxl-y += vphb.o api.o cxllib.o cxl-$(CONFIG_PPC_PSERIES) += flash.o guest.o of.o hcalls.o cxl-$(CONFIG_DEBUG_FS) += debugfs.o obj-$(CONFIG_CXL) += cxl.o diff --git a/drivers/misc/cxl/api.c b/drivers/misc/cxl/api.c index 8fd5ec4d6042..750470ef2049 100644 --- a/drivers/misc/cxl/api.c +++ b/drivers/misc/cxl/api.c @@ -11,7 +11,6 @@ #include <linux/slab.h> #include <linux/file.h> #include <misc/cxl.h> -#include <linux/msi.h> #include <linux/module.h> #include <linux/mount.h> #include <linux/sched/mm.h> @@ -168,21 +167,6 @@ static irq_hw_number_t cxl_find_afu_irq(struct cxl_context *ctx, int num) return 0; } -int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq) -{ - if (*ctx == NULL || *afu_irq == 0) { - *afu_irq = 1; - *ctx = cxl_get_context(pdev); - } else { - (*afu_irq)++; - if (*afu_irq > cxl_get_max_irqs_per_process(pdev)) { - *ctx = list_next_entry(*ctx, extra_irq_contexts); - *afu_irq = 1; - } - } - return cxl_find_afu_irq(*ctx, *afu_irq); -} -/* Exported via cxl_base */ int cxl_set_priv(struct cxl_context *ctx, void *priv) { @@ -310,7 +294,6 @@ int cxl_start_context(struct cxl_context *ctx, u64 wed, if (task) { ctx->pid = get_task_pid(task, PIDTYPE_PID); kernel = false; - ctx->real_mode = false; /* acquire a reference to the task's mm */ ctx->mm = get_task_mm(current); @@ -374,24 +357,6 @@ void cxl_set_master(struct cxl_context *ctx) } EXPORT_SYMBOL_GPL(cxl_set_master); -int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode) -{ - if (ctx->status == STARTED) { - /* - * We could potentially update the PE and issue an update LLCMD - * to support this, but it doesn't seem to have a good use case - * since it's trivial to just create a second kernel context - * with different translation modes, so until someone convinces - * me otherwise: - */ - return -EBUSY; - } - - ctx->real_mode = real_mode; - return 0; -} -EXPORT_SYMBOL_GPL(cxl_set_translation_mode); - /* wrappers around afu_* file ops which are EXPORTED */ int cxl_fd_open(struct inode *inode, struct file *file) { @@ -573,100 +538,3 @@ ssize_t cxl_read_adapter_vpd(struct pci_dev *dev, void *buf, size_t count) return cxl_ops->read_adapter_vpd(afu->adapter, buf, count); } EXPORT_SYMBOL_GPL(cxl_read_adapter_vpd); - -int cxl_set_max_irqs_per_process(struct pci_dev *dev, int irqs) -{ - struct cxl_afu *afu = cxl_pci_to_afu(dev); - if (IS_ERR(afu)) - return -ENODEV; - - if (irqs > afu->adapter->user_irqs) - return -EINVAL; - - /* Limit user_irqs to prevent the user increasing this via sysfs */ - afu->adapter->user_irqs = irqs; - afu->irqs_max = irqs; - - return 0; -} -EXPORT_SYMBOL_GPL(cxl_set_max_irqs_per_process); - -int cxl_get_max_irqs_per_process(struct pci_dev *dev) -{ - struct cxl_afu *afu = cxl_pci_to_afu(dev); - if (IS_ERR(afu)) - return -ENODEV; - - return afu->irqs_max; -} -EXPORT_SYMBOL_GPL(cxl_get_max_irqs_per_process); - -/* - * This is a special interrupt allocation routine called from the PHB's MSI - * setup function. When capi interrupts are allocated in this manner they must - * still be associated with a running context, but since the MSI APIs have no - * way to specify this we use the default context associated with the device. - * - * The Mellanox CX4 has a hardware limitation that restricts the maximum AFU - * interrupt number, so in order to overcome this their driver informs us of - * the restriction by setting the maximum interrupts per context, and we - * allocate additional contexts as necessary so that we can keep the AFU - * interrupt number within the supported range. - */ -int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) -{ - struct cxl_context *ctx, *new_ctx, *default_ctx; - int remaining; - int rc; - - ctx = default_ctx = cxl_get_context(pdev); - if (WARN_ON(!default_ctx)) - return -ENODEV; - - remaining = nvec; - while (remaining > 0) { - rc = cxl_allocate_afu_irqs(ctx, min(remaining, ctx->afu->irqs_max)); - if (rc) { - pr_warn("%s: Failed to find enough free MSIs\n", pci_name(pdev)); - return rc; - } - remaining -= ctx->afu->irqs_max; - - if (ctx != default_ctx && default_ctx->status == STARTED) { - WARN_ON(cxl_start_context(ctx, - be64_to_cpu(default_ctx->elem->common.wed), - NULL)); - } - - if (remaining > 0) { - new_ctx = cxl_dev_context_init(pdev); - if (IS_ERR(new_ctx)) { - pr_warn("%s: Failed to allocate enough contexts for MSIs\n", pci_name(pdev)); - return -ENOSPC; - } - list_add(&new_ctx->extra_irq_contexts, &ctx->extra_irq_contexts); - ctx = new_ctx; - } - } - - return 0; -} -/* Exported via cxl_base */ - -void _cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev) -{ - struct cxl_context *ctx, *pos, *tmp; - - ctx = cxl_get_context(pdev); - if (WARN_ON(!ctx)) - return; - - cxl_free_afu_irqs(ctx); - list_for_each_entry_safe(pos, tmp, &ctx->extra_irq_contexts, extra_irq_contexts) { - cxl_stop_context(pos); - cxl_free_afu_irqs(pos); - list_del(&pos->extra_irq_contexts); - cxl_release_context(pos); - } -} -/* Exported via cxl_base */ diff --git a/drivers/misc/cxl/base.c b/drivers/misc/cxl/base.c index cd54ce6f6230..7557835cdfcd 100644 --- a/drivers/misc/cxl/base.c +++ b/drivers/misc/cxl/base.c @@ -106,89 +106,6 @@ int cxl_update_properties(struct device_node *dn, } EXPORT_SYMBOL_GPL(cxl_update_properties); -/* - * API calls into the driver that may be called from the PHB code and must be - * built in. - */ -bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) -{ - bool ret; - struct cxl_calls *calls; - - calls = cxl_calls_get(); - if (!calls) - return false; - - ret = calls->cxl_pci_associate_default_context(dev, afu); - - cxl_calls_put(calls); - - return ret; -} -EXPORT_SYMBOL_GPL(cxl_pci_associate_default_context); - -void cxl_pci_disable_device(struct pci_dev *dev) -{ - struct cxl_calls *calls; - - calls = cxl_calls_get(); - if (!calls) - return; - - calls->cxl_pci_disable_device(dev); - - cxl_calls_put(calls); -} -EXPORT_SYMBOL_GPL(cxl_pci_disable_device); - -int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq) -{ - int ret; - struct cxl_calls *calls; - - calls = cxl_calls_get(); - if (!calls) - return -EBUSY; - - ret = calls->cxl_next_msi_hwirq(pdev, ctx, afu_irq); - - cxl_calls_put(calls); - - return ret; -} -EXPORT_SYMBOL_GPL(cxl_next_msi_hwirq); - -int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) -{ - int ret; - struct cxl_calls *calls; - - calls = cxl_calls_get(); - if (!calls) - return false; - - ret = calls->cxl_cx4_setup_msi_irqs(pdev, nvec, type); - - cxl_calls_put(calls); - - return ret; -} -EXPORT_SYMBOL_GPL(cxl_cx4_setup_msi_irqs); - -void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev) -{ - struct cxl_calls *calls; - - calls = cxl_calls_get(); - if (!calls) - return; - - calls->cxl_cx4_teardown_msi_irqs(pdev); - - cxl_calls_put(calls); -} -EXPORT_SYMBOL_GPL(cxl_cx4_teardown_msi_irqs); - static int __init cxl_base_init(void) { struct device_node *np; diff --git a/drivers/misc/cxl/context.c b/drivers/misc/cxl/context.c index c6ec872800a2..5fe529b43ebe 100644 --- a/drivers/misc/cxl/context.c +++ b/drivers/misc/cxl/context.c @@ -74,7 +74,6 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master) ctx->pending_afu_err = false; INIT_LIST_HEAD(&ctx->irq_names); - INIT_LIST_HEAD(&ctx->extra_irq_contexts); /* * When we have to destroy all contexts in cxl_context_detach_all() we @@ -96,7 +95,7 @@ int cxl_context_init(struct cxl_context *ctx, struct cxl_afu *afu, bool master) */ mutex_lock(&afu->contexts_lock); idr_preload(GFP_KERNEL); - i = idr_alloc(&ctx->afu->contexts_idr, ctx, ctx->afu->adapter->min_pe, + i = idr_alloc(&ctx->afu->contexts_idr, ctx, 0, ctx->afu->num_procs, GFP_NOWAIT); idr_preload_end(); mutex_unlock(&afu->contexts_lock); diff --git a/drivers/misc/cxl/cxl.h b/drivers/misc/cxl/cxl.h index 918d4fb742d1..d1d927ccb589 100644 --- a/drivers/misc/cxl/cxl.h +++ b/drivers/misc/cxl/cxl.h @@ -93,11 +93,6 @@ static const cxl_p1_reg_t CXL_PSL_FIR_CNTL = {0x0148}; static const cxl_p1_reg_t CXL_PSL_DSNDCTL = {0x0150}; static const cxl_p1_reg_t CXL_PSL_SNWRALLOC = {0x0158}; static const cxl_p1_reg_t CXL_PSL_TRACE = {0x0170}; -/* XSL registers (Mellanox CX4) */ -static const cxl_p1_reg_t CXL_XSL_Timebase = {0x0100}; -static const cxl_p1_reg_t CXL_XSL_TB_CTLSTAT = {0x0108}; -static const cxl_p1_reg_t CXL_XSL_FEC = {0x0158}; -static const cxl_p1_reg_t CXL_XSL_DSNCTL = {0x0168}; /* PSL registers - CAIA 2 */ static const cxl_p1_reg_t CXL_PSL9_CONTROL = {0x0020}; static const cxl_p1_reg_t CXL_XSL9_INV = {0x0110}; @@ -613,7 +608,6 @@ struct cxl_context { bool pe_inserted; bool master; bool kernel; - bool real_mode; bool pending_irq; bool pending_fault; bool pending_afu_err; @@ -624,14 +618,6 @@ struct cxl_context { struct rcu_head rcu; - /* - * Only used when more interrupts are allocated via - * pci_enable_msix_range than are supported in the default context, to - * use additional contexts to overcome the limitation. i.e. Mellanox - * CX4 only: - */ - struct list_head extra_irq_contexts; - struct mm_struct *mm; u16 tidr; @@ -704,7 +690,6 @@ struct cxl { struct bin_attribute cxl_attr; int adapter_num; int user_irqs; - int min_pe; u64 ps_size; u16 psl_rev; u16 base_image; @@ -865,32 +850,12 @@ static inline bool cxl_is_power9(void) return false; } -static inline bool cxl_is_power9_dd1(void) -{ - if ((pvr_version_is(PVR_POWER9)) && - cpu_has_feature(CPU_FTR_POWER9_DD1)) - return true; - return false; -} - ssize_t cxl_pci_afu_read_err_buffer(struct cxl_afu *afu, char *buf, loff_t off, size_t count); -/* Internal functions wrapped in cxl_base to allow PHB to call them */ -bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu); -void _cxl_pci_disable_device(struct pci_dev *dev); -int _cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq); -int _cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type); -void _cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev); struct cxl_calls { void (*cxl_slbia)(struct mm_struct *mm); - bool (*cxl_pci_associate_default_context)(struct pci_dev *dev, struct cxl_afu *afu); - void (*cxl_pci_disable_device)(struct pci_dev *dev); - int (*cxl_next_msi_hwirq)(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq); - int (*cxl_cx4_setup_msi_irqs)(struct pci_dev *pdev, int nvec, int type); - void (*cxl_cx4_teardown_msi_irqs)(struct pci_dev *pdev); - struct module *owner; }; int register_cxl_calls(struct cxl_calls *calls); @@ -955,7 +920,6 @@ int cxl_debugfs_afu_add(struct cxl_afu *afu); void cxl_debugfs_afu_remove(struct cxl_afu *afu); void cxl_debugfs_add_adapter_regs_psl9(struct cxl *adapter, struct dentry *dir); void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir); -void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir); void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir); void cxl_debugfs_add_afu_regs_psl8(struct cxl_afu *afu, struct dentry *dir); @@ -998,11 +962,6 @@ static inline void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, { } -static inline void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, - struct dentry *dir) -{ -} - static inline void cxl_debugfs_add_afu_regs_psl9(struct cxl_afu *afu, struct dentry *dir) { } diff --git a/drivers/misc/cxl/cxllib.c b/drivers/misc/cxl/cxllib.c index 0bc7c31cf739..5a3f91255258 100644 --- a/drivers/misc/cxl/cxllib.c +++ b/drivers/misc/cxl/cxllib.c @@ -102,10 +102,6 @@ int cxllib_get_xsl_config(struct pci_dev *dev, struct cxllib_xsl_config *cfg) rc = cxl_get_xsl9_dsnctl(dev, capp_unit_id, &cfg->dsnctl); if (rc) return rc; - if (cpu_has_feature(CPU_FTR_POWER9_DD1)) { - /* workaround for DD1 - nbwind = capiind */ - cfg->dsnctl |= ((u64)0x02 << (63-47)); - } cfg->version = CXL_XSL_CONFIG_CURRENT_VERSION; cfg->log_bar_size = CXL_CAPI_WINDOW_LOG_SIZE; diff --git a/drivers/misc/cxl/debugfs.c b/drivers/misc/cxl/debugfs.c index 1643850d2302..a1921d81593a 100644 --- a/drivers/misc/cxl/debugfs.c +++ b/drivers/misc/cxl/debugfs.c @@ -58,11 +58,6 @@ void cxl_debugfs_add_adapter_regs_psl8(struct cxl *adapter, struct dentry *dir) debugfs_create_io_x64("trace", S_IRUSR | S_IWUSR, dir, _cxl_p1_addr(adapter, CXL_PSL_TRACE)); } -void cxl_debugfs_add_adapter_regs_xsl(struct cxl *adapter, struct dentry *dir) -{ - debugfs_create_io_x64("fec", S_IRUSR, dir, _cxl_p1_addr(adapter, CXL_XSL_FEC)); -} - int cxl_debugfs_adapter_add(struct cxl *adapter) { struct dentry *dir; diff --git a/drivers/misc/cxl/fault.c b/drivers/misc/cxl/fault.c index 70dbb6de102c..d45f3e6b17d2 100644 --- a/drivers/misc/cxl/fault.c +++ b/drivers/misc/cxl/fault.c @@ -33,7 +33,7 @@ static bool sste_matches(struct cxl_sste *sste, struct copro_slb *slb) * This finds a free SSTE for the given SLB, or returns NULL if it's already in * the segment table. */ -static struct cxl_sste* find_free_sste(struct cxl_context *ctx, +static struct cxl_sste *find_free_sste(struct cxl_context *ctx, struct copro_slb *slb) { struct cxl_sste *primary, *sste, *ret = NULL; diff --git a/drivers/misc/cxl/guest.c b/drivers/misc/cxl/guest.c index 4644f16606a3..3bc0c15d4d85 100644 --- a/drivers/misc/cxl/guest.c +++ b/drivers/misc/cxl/guest.c @@ -623,9 +623,6 @@ static int guest_attach_process(struct cxl_context *ctx, bool kernel, u64 wed, u { pr_devel("in %s\n", __func__); - if (ctx->real_mode) - return -EPERM; - ctx->kernel = kernel; if (ctx->afu->current_mode == CXL_MODE_DIRECTED) return attach_afu_directed(ctx, wed, amr); @@ -916,11 +913,6 @@ static int afu_properties_look_ok(struct cxl_afu *afu) return -EINVAL; } - if (afu->crs_len < 0) { - dev_err(&afu->dev, "Unexpected configuration record size value\n"); - return -EINVAL; - } - return 0; } diff --git a/drivers/misc/cxl/main.c b/drivers/misc/cxl/main.c index c1ba0d42cbc8..f35406be465a 100644 --- a/drivers/misc/cxl/main.c +++ b/drivers/misc/cxl/main.c @@ -104,11 +104,6 @@ static inline void cxl_slbia_core(struct mm_struct *mm) static struct cxl_calls cxl_calls = { .cxl_slbia = cxl_slbia_core, - .cxl_pci_associate_default_context = _cxl_pci_associate_default_context, - .cxl_pci_disable_device = _cxl_pci_disable_device, - .cxl_next_msi_hwirq = _cxl_next_msi_hwirq, - .cxl_cx4_setup_msi_irqs = _cxl_cx4_setup_msi_irqs, - .cxl_cx4_teardown_msi_irqs = _cxl_cx4_teardown_msi_irqs, .owner = THIS_MODULE, }; @@ -287,7 +282,7 @@ int cxl_adapter_context_get(struct cxl *adapter) int rc; rc = atomic_inc_unless_negative(&adapter->contexts_num); - return rc >= 0 ? 0 : -EBUSY; + return rc ? 0 : -EBUSY; } void cxl_adapter_context_put(struct cxl *adapter) diff --git a/drivers/misc/cxl/native.c b/drivers/misc/cxl/native.c index 98f867fcef24..c9d5d82dce8e 100644 --- a/drivers/misc/cxl/native.c +++ b/drivers/misc/cxl/native.c @@ -605,6 +605,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9) sr |= CXL_PSL_SR_An_MP; if (mfspr(SPRN_LPCR) & LPCR_TC) sr |= CXL_PSL_SR_An_TC; + if (kernel) { if (!real_mode) sr |= CXL_PSL_SR_An_R; @@ -629,7 +630,7 @@ u64 cxl_calculate_sr(bool master, bool kernel, bool real_mode, bool p9) static u64 calculate_sr(struct cxl_context *ctx) { - return cxl_calculate_sr(ctx->master, ctx->kernel, ctx->real_mode, + return cxl_calculate_sr(ctx->master, ctx->kernel, false, cxl_is_power9()); } diff --git a/drivers/misc/cxl/pci.c b/drivers/misc/cxl/pci.c index 429d6de1dde7..b66d832d3233 100644 --- a/drivers/misc/cxl/pci.c +++ b/drivers/misc/cxl/pci.c @@ -55,8 +55,6 @@ pci_read_config_byte(dev, vsec + 0xa, dest) #define CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val) \ pci_write_config_byte(dev, vsec + 0xa, val) -#define CXL_WRITE_VSEC_MODE_CONTROL_BUS(bus, devfn, vsec, val) \ - pci_bus_write_config_byte(bus, devfn, vsec + 0xa, val) #define CXL_VSEC_PROTOCOL_MASK 0xe0 #define CXL_VSEC_PROTOCOL_1024TB 0x80 #define CXL_VSEC_PROTOCOL_512TB 0x40 @@ -465,23 +463,21 @@ int cxl_get_xsl9_dsnctl(struct pci_dev *dev, u64 capp_unit_id, u64 *reg) /* nMMU_ID Defaults to: b’000001001’*/ xsl_dsnctl |= ((u64)0x09 << (63-28)); - if (!(cxl_is_power9_dd1())) { - /* - * Used to identify CAPI packets which should be sorted into - * the Non-Blocking queues by the PHB. This field should match - * the PHB PBL_NBW_CMPM register - * nbwind=0x03, bits [57:58], must include capi indicator. - * Not supported on P9 DD1. - */ - xsl_dsnctl |= (nbwind << (63-55)); + /* + * Used to identify CAPI packets which should be sorted into + * the Non-Blocking queues by the PHB. This field should match + * the PHB PBL_NBW_CMPM register + * nbwind=0x03, bits [57:58], must include capi indicator. + * Not supported on P9 DD1. + */ + xsl_dsnctl |= (nbwind << (63-55)); - /* - * Upper 16b address bits of ASB_Notify messages sent to the - * system. Need to match the PHB’s ASN Compare/Mask Register. - * Not supported on P9 DD1. - */ - xsl_dsnctl |= asnind; - } + /* + * Upper 16b address bits of ASB_Notify messages sent to the + * system. Need to match the PHB’s ASN Compare/Mask Register. + * Not supported on P9 DD1. + */ + xsl_dsnctl |= asnind; *reg = xsl_dsnctl; return 0; @@ -539,15 +535,8 @@ static int init_implementation_adapter_regs_psl9(struct cxl *adapter, /* Snoop machines */ cxl_p1_write(adapter, CXL_PSL9_APCDEDALLOC, 0x800F000200000000ULL); - if (cxl_is_power9_dd1()) { - /* Disabling deadlock counter CAR */ - cxl_p1_write(adapter, CXL_PSL9_GP_CT, 0x0020000000000001ULL); - /* Enable NORST */ - cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0x8000000000000000ULL); - } else { - /* Enable NORST and DD2 features */ - cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0xC000000000000000ULL); - } + /* Enable NORST and DD2 features */ + cxl_p1_write(adapter, CXL_PSL9_DEBUG, 0xC000000000000000ULL); /* * Check if PSL has data-cache. We need to flush adapter datacache @@ -595,27 +584,7 @@ static int init_implementation_adapter_regs_psl8(struct cxl *adapter, struct pci return 0; } -static int init_implementation_adapter_regs_xsl(struct cxl *adapter, struct pci_dev *dev) -{ - u64 xsl_dsnctl; - u64 chipid; - u32 phb_index; - u64 capp_unit_id; - int rc; - - rc = cxl_calc_capp_routing(dev, &chipid, &phb_index, &capp_unit_id); - if (rc) - return rc; - - /* Tell XSL where to route data to */ - xsl_dsnctl = 0x0000600000000000ULL | (chipid << (63-5)); - xsl_dsnctl |= (capp_unit_id << (63-13)); - cxl_p1_write(adapter, CXL_XSL_DSNCTL, xsl_dsnctl); - - return 0; -} - -/* PSL & XSL */ +/* PSL */ #define TBSYNC_CAL(n) (((u64)n & 0x7) << (63-3)) #define TBSYNC_CNT(n) (((u64)n & 0x7) << (63-6)) /* For the PSL this is a multiple for 0 < n <= 7: */ @@ -627,21 +596,6 @@ static void write_timebase_ctrl_psl8(struct cxl *adapter) TBSYNC_CNT(2 * PSL_2048_250MHZ_CYCLES)); } -/* XSL */ -#define TBSYNC_ENA (1ULL << 63) -/* For the XSL this is 2**n * 2000 clocks for 0 < n <= 6: */ -#define XSL_2000_CLOCKS 1 -#define XSL_4000_CLOCKS 2 -#define XSL_8000_CLOCKS 3 - -static void write_timebase_ctrl_xsl(struct cxl *adapter) -{ - cxl_p1_write(adapter, CXL_XSL_TB_CTLSTAT, - TBSYNC_ENA | - TBSYNC_CAL(3) | - TBSYNC_CNT(XSL_4000_CLOCKS)); -} - static u64 timebase_read_psl9(struct cxl *adapter) { return cxl_p1_read(adapter, CXL_PSL9_Timebase); @@ -652,11 +606,6 @@ static u64 timebase_read_psl8(struct cxl *adapter) return cxl_p1_read(adapter, CXL_PSL_Timebase); } -static u64 timebase_read_xsl(struct cxl *adapter) -{ - return cxl_p1_read(adapter, CXL_XSL_Timebase); -} - static void cxl_setup_psl_timebase(struct cxl *adapter, struct pci_dev *dev) { struct device_node *np; @@ -800,234 +749,36 @@ static int setup_cxl_bars(struct pci_dev *dev) return 0; } -#ifdef CONFIG_CXL_BIMODAL - -struct cxl_switch_work { - struct pci_dev *dev; - struct work_struct work; - int vsec; - int mode; -}; - -static void switch_card_to_cxl(struct work_struct *work) +/* pciex node: ibm,opal-m64-window = <0x3d058 0x0 0x3d058 0x0 0x8 0x0>; */ +static int switch_card_to_cxl(struct pci_dev *dev) { - struct cxl_switch_work *switch_work = - container_of(work, struct cxl_switch_work, work); - struct pci_dev *dev = switch_work->dev; - struct pci_bus *bus = dev->bus; - struct pci_controller *hose = pci_bus_to_host(bus); - struct pci_dev *bridge; - struct pnv_php_slot *php_slot; - unsigned int devfn; + int vsec; u8 val; int rc; - dev_info(&bus->dev, "cxl: Preparing for mode switch...\n"); - bridge = list_first_entry_or_null(&hose->bus->devices, struct pci_dev, - bus_list); - if (!bridge) { - dev_WARN(&bus->dev, "cxl: Couldn't find root port!\n"); - goto err_dev_put; - } - - php_slot = pnv_php_find_slot(pci_device_to_OF_node(bridge)); - if (!php_slot) { - dev_err(&bus->dev, "cxl: Failed to find slot hotplug " - "information. You may need to upgrade " - "skiboot. Aborting.\n"); - goto err_dev_put; - } - - rc = CXL_READ_VSEC_MODE_CONTROL(dev, switch_work->vsec, &val); - if (rc) { - dev_err(&bus->dev, "cxl: Failed to read CAPI mode control: %i\n", rc); - goto err_dev_put; - } - devfn = dev->devfn; - - /* Release the reference obtained in cxl_check_and_switch_mode() */ - pci_dev_put(dev); - - dev_dbg(&bus->dev, "cxl: Removing PCI devices from kernel\n"); - pci_lock_rescan_remove(); - pci_hp_remove_devices(bridge->subordinate); - pci_unlock_rescan_remove(); - - /* Switch the CXL protocol on the card */ - if (switch_work->mode == CXL_BIMODE_CXL) { - dev_info(&bus->dev, "cxl: Switching card to CXL mode\n"); - val &= ~CXL_VSEC_PROTOCOL_MASK; - val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE; - rc = pnv_cxl_enable_phb_kernel_api(hose, true); - if (rc) { - dev_err(&bus->dev, "cxl: Failed to enable kernel API" - " on real PHB, aborting\n"); - goto err_free_work; - } - } else { - dev_WARN(&bus->dev, "cxl: Switching card to PCI mode not supported!\n"); - goto err_free_work; - } - - rc = CXL_WRITE_VSEC_MODE_CONTROL_BUS(bus, devfn, switch_work->vsec, val); - if (rc) { - dev_err(&bus->dev, "cxl: Failed to configure CXL protocol: %i\n", rc); - goto err_free_work; - } + dev_info(&dev->dev, "switch card to CXL\n"); - /* - * The CAIA spec (v1.1, Section 10.6 Bi-modal Device Support) states - * we must wait 100ms after this mode switch before touching PCIe config - * space. - */ - msleep(100); - - /* - * Hot reset to cause the card to come back in cxl mode. A - * OPAL_RESET_PCI_LINK would be sufficient, but currently lacks support - * in skiboot, so we use a hot reset instead. - * - * We call pci_set_pcie_reset_state() on the bridge, as a CAPI card is - * guaranteed to sit directly under the root port, and setting the reset - * state on a device directly under the root port is equivalent to doing - * it on the root port iself. - */ - dev_info(&bus->dev, "cxl: Configuration write complete, resetting card\n"); - pci_set_pcie_reset_state(bridge, pcie_hot_reset); - pci_set_pcie_reset_state(bridge, pcie_deassert_reset); - - dev_dbg(&bus->dev, "cxl: Offlining slot\n"); - rc = pnv_php_set_slot_power_state(&php_slot->slot, OPAL_PCI_SLOT_OFFLINE); - if (rc) { - dev_err(&bus->dev, "cxl: OPAL offlining call failed: %i\n", rc); - goto err_free_work; - } - - dev_dbg(&bus->dev, "cxl: Onlining and probing slot\n"); - rc = pnv_php_set_slot_power_state(&php_slot->slot, OPAL_PCI_SLOT_ONLINE); - if (rc) { - dev_err(&bus->dev, "cxl: OPAL onlining call failed: %i\n", rc); - goto err_free_work; - } - - pci_lock_rescan_remove(); - pci_hp_add_devices(bridge->subordinate); - pci_unlock_rescan_remove(); - - dev_info(&bus->dev, "cxl: CAPI mode switch completed\n"); - kfree(switch_work); - return; - -err_dev_put: - /* Release the reference obtained in cxl_check_and_switch_mode() */ - pci_dev_put(dev); -err_free_work: - kfree(switch_work); -} - -int cxl_check_and_switch_mode(struct pci_dev *dev, int mode, int vsec) -{ - struct cxl_switch_work *work; - u8 val; - int rc; - - if (!cpu_has_feature(CPU_FTR_HVMODE)) + if (!(vsec = find_cxl_vsec(dev))) { + dev_err(&dev->dev, "ABORTING: CXL VSEC not found!\n"); return -ENODEV; - - if (!vsec) { - vsec = find_cxl_vsec(dev); - if (!vsec) { - dev_info(&dev->dev, "CXL VSEC not found\n"); - return -ENODEV; - } } - rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val); - if (rc) { - dev_err(&dev->dev, "Failed to read current mode control: %i", rc); + if ((rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val))) { + dev_err(&dev->dev, "failed to read current mode control: %i", rc); return rc; } - - if (mode == CXL_BIMODE_PCI) { - if (!(val & CXL_VSEC_PROTOCOL_ENABLE)) { - dev_info(&dev->dev, "Card is already in PCI mode\n"); - return 0; - } - /* - * TODO: Before it's safe to switch the card back to PCI mode - * we need to disable the CAPP and make sure any cachelines the - * card holds have been flushed out. Needs skiboot support. - */ - dev_WARN(&dev->dev, "CXL mode switch to PCI unsupported!\n"); - return -EIO; - } - - if (val & CXL_VSEC_PROTOCOL_ENABLE) { - dev_info(&dev->dev, "Card is already in CXL mode\n"); - return 0; + val &= ~CXL_VSEC_PROTOCOL_MASK; + val |= CXL_VSEC_PROTOCOL_256TB | CXL_VSEC_PROTOCOL_ENABLE; + if ((rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val))) { + dev_err(&dev->dev, "failed to enable CXL protocol: %i", rc); + return rc; } - - dev_info(&dev->dev, "Card is in PCI mode, scheduling kernel thread " - "to switch to CXL mode\n"); - - work = kmalloc(sizeof(struct cxl_switch_work), GFP_KERNEL); - if (!work) - return -ENOMEM; - - pci_dev_get(dev); - work->dev = dev; - work->vsec = vsec; - work->mode = mode; - INIT_WORK(&work->work, switch_card_to_cxl); - - schedule_work(&work->work); - /* - * We return a failure now to abort the driver init. Once the - * link has been cycled and the card is in cxl mode we will - * come back (possibly using the generic cxl driver), but - * return success as the card should then be in cxl mode. - * - * TODO: What if the card comes back in PCI mode even after - * the switch? Don't want to spin endlessly. + * The CAIA spec (v0.12 11.6 Bi-modal Device Support) states + * we must wait 100ms after this mode switch before touching + * PCIe config space. */ - return -EBUSY; -} -EXPORT_SYMBOL_GPL(cxl_check_and_switch_mode); - -#endif /* CONFIG_CXL_BIMODAL */ - -static int setup_cxl_protocol_area(struct pci_dev *dev) -{ - u8 val; - int rc; - int vsec = find_cxl_vsec(dev); - - if (!vsec) { - dev_info(&dev->dev, "CXL VSEC not found\n"); - return -ENODEV; - } - - rc = CXL_READ_VSEC_MODE_CONTROL(dev, vsec, &val); - if (rc) { - dev_err(&dev->dev, "Failed to read current mode control: %i\n", rc); - return rc; - } - - if (!(val & CXL_VSEC_PROTOCOL_ENABLE)) { - dev_err(&dev->dev, "Card not in CAPI mode!\n"); - return -EIO; - } - - if ((val & CXL_VSEC_PROTOCOL_MASK) != CXL_VSEC_PROTOCOL_256TB) { - val &= ~CXL_VSEC_PROTOCOL_MASK; - val |= CXL_VSEC_PROTOCOL_256TB; - rc = CXL_WRITE_VSEC_MODE_CONTROL(dev, vsec, val); - if (rc) { - dev_err(&dev->dev, "Failed to set CXL protocol area: %i\n", rc); - return rc; - } - } + msleep(100); return 0; } @@ -1724,7 +1475,7 @@ static int cxl_configure_adapter(struct cxl *adapter, struct pci_dev *dev) if ((rc = setup_cxl_bars(dev))) return rc; - if ((rc = setup_cxl_protocol_area(dev))) + if ((rc = switch_card_to_cxl(dev))) return rc; if ((rc = cxl_update_image_control(adapter))) @@ -1871,37 +1622,14 @@ static const struct cxl_service_layer_ops psl8_ops = { .needs_reset_before_disable = true, }; -static const struct cxl_service_layer_ops xsl_ops = { - .adapter_regs_init = init_implementation_adapter_regs_xsl, - .invalidate_all = cxl_invalidate_all_psl8, - .sanitise_afu_regs = sanitise_afu_regs_psl8, - .handle_interrupt = cxl_irq_psl8, - .fail_irq = cxl_fail_irq_psl, - .activate_dedicated_process = cxl_activate_dedicated_process_psl8, - .attach_afu_directed = cxl_attach_afu_directed_psl8, - .attach_dedicated_process = cxl_attach_dedicated_process_psl8, - .update_dedicated_ivtes = cxl_update_dedicated_ivtes_psl8, - .debugfs_add_adapter_regs = cxl_debugfs_add_adapter_regs_xsl, - .write_timebase_ctrl = write_timebase_ctrl_xsl, - .timebase_read = timebase_read_xsl, - .capi_mode = OPAL_PHB_CAPI_MODE_DMA, -}; - static void set_sl_ops(struct cxl *adapter, struct pci_dev *dev) { - if (dev->vendor == PCI_VENDOR_ID_MELLANOX && dev->device == 0x1013) { - /* Mellanox CX-4 */ - dev_info(&dev->dev, "Device uses an XSL\n"); - adapter->native->sl_ops = &xsl_ops; - adapter->min_pe = 1; /* Workaround for CX-4 hardware bug */ + if (cxl_is_power8()) { + dev_info(&dev->dev, "Device uses a PSL8\n"); + adapter->native->sl_ops = &psl8_ops; } else { - if (cxl_is_power8()) { - dev_info(&dev->dev, "Device uses a PSL8\n"); - adapter->native->sl_ops = &psl8_ops; - } else { - dev_info(&dev->dev, "Device uses a PSL9\n"); - adapter->native->sl_ops = &psl9_ops; - } + dev_info(&dev->dev, "Device uses a PSL9\n"); + adapter->native->sl_ops = &psl9_ops; } } @@ -2008,43 +1736,6 @@ int cxl_slot_is_switched(struct pci_dev *dev) return (depth > CXL_MAX_PCIEX_PARENT); } -bool cxl_slot_is_supported(struct pci_dev *dev, int flags) -{ - if (!cpu_has_feature(CPU_FTR_HVMODE)) - return false; - - if ((flags & CXL_SLOT_FLAG_DMA) && (!pvr_version_is(PVR_POWER8NVL))) { - /* - * CAPP DMA mode is technically supported on regular P8, but - * will EEH if the card attempts to access memory < 4GB, which - * we cannot realistically avoid. We might be able to work - * around the issue, but until then return unsupported: - */ - return false; - } - - if (cxl_slot_is_switched(dev)) - return false; - - /* - * XXX: This gets a little tricky on regular P8 (not POWER8NVL) since - * the CAPP can be connected to PHB 0, 1 or 2 on a first come first - * served basis, which is racy to check from here. If we need to - * support this in future we might need to consider having this - * function effectively reserve it ahead of time. - * - * Currently, the only user of this API is the Mellanox CX4, which is - * only supported on P8NVL due to the above mentioned limitation of - * CAPP DMA mode and therefore does not need to worry about this. If the - * issue with CAPP DMA mode is later worked around on P8 we might need - * to revisit this. - */ - - return true; -} -EXPORT_SYMBOL_GPL(cxl_slot_is_supported); - - static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id) { struct cxl *adapter; @@ -2086,9 +1777,6 @@ static int cxl_probe(struct pci_dev *dev, const struct pci_device_id *id) dev_err(&dev->dev, "AFU %i failed to start: %i\n", slice, rc); } - if (pnv_pci_on_cxl_phb(dev) && adapter->slices >= 1) - pnv_cxl_phb_set_peer_afu(dev, adapter->afu[0]); - return 0; } diff --git a/drivers/misc/cxl/phb.c b/drivers/misc/cxl/phb.c deleted file mode 100644 index 6ec69ada19f4..000000000000 --- a/drivers/misc/cxl/phb.c +++ /dev/null @@ -1,44 +0,0 @@ -/* - * Copyright 2014-2016 IBM Corp. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - */ - -#include <linux/pci.h> -#include "cxl.h" - -bool _cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) -{ - struct cxl_context *ctx; - - /* - * Allocate a context to do cxl things to. This is used for interrupts - * in the peer model using a real phb, and if we eventually do DMA ops - * in the virtual phb, we'll need a default context to attach them to. - */ - ctx = cxl_dev_context_init(dev); - if (IS_ERR(ctx)) - return false; - dev->dev.archdata.cxl_ctx = ctx; - - return (cxl_ops->afu_check_and_enable(afu) == 0); -} -/* exported via cxl_base */ - -void _cxl_pci_disable_device(struct pci_dev *dev) -{ - struct cxl_context *ctx = cxl_get_context(dev); - - if (ctx) { - if (ctx->status == STARTED) { - dev_err(&dev->dev, "Default context started\n"); - return; - } - dev->dev.archdata.cxl_ctx = NULL; - cxl_release_context(ctx); - } -} -/* exported via cxl_base */ diff --git a/drivers/misc/cxl/vphb.c b/drivers/misc/cxl/vphb.c index 7fd0bdc1436a..7908633d9204 100644 --- a/drivers/misc/cxl/vphb.c +++ b/drivers/misc/cxl/vphb.c @@ -9,7 +9,6 @@ #include <linux/pci.h> #include <misc/cxl.h> -#include <asm/pnv-pci.h> #include "cxl.h" static int cxl_dma_set_mask(struct pci_dev *pdev, u64 dma_mask) @@ -45,6 +44,7 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev) { struct pci_controller *phb; struct cxl_afu *afu; + struct cxl_context *ctx; phb = pci_bus_to_host(dev->bus); afu = (struct cxl_afu *)phb->private_data; @@ -57,7 +57,30 @@ static bool cxl_pci_enable_device_hook(struct pci_dev *dev) set_dma_ops(&dev->dev, &dma_nommu_ops); set_dma_offset(&dev->dev, PAGE_OFFSET); - return _cxl_pci_associate_default_context(dev, afu); + /* + * Allocate a context to do cxl things too. If we eventually do real + * DMA ops, we'll need a default context to attach them to + */ + ctx = cxl_dev_context_init(dev); + if (IS_ERR(ctx)) + return false; + dev->dev.archdata.cxl_ctx = ctx; + + return (cxl_ops->afu_check_and_enable(afu) == 0); +} + +static void cxl_pci_disable_device(struct pci_dev *dev) +{ + struct cxl_context *ctx = cxl_get_context(dev); + + if (ctx) { + if (ctx->status == STARTED) { + dev_err(&dev->dev, "Default context started\n"); + return; + } + dev->dev.archdata.cxl_ctx = NULL; + cxl_release_context(ctx); + } } static resource_size_t cxl_pci_window_alignment(struct pci_bus *bus, @@ -191,8 +214,8 @@ static struct pci_controller_ops cxl_pci_controller_ops = { .probe_mode = cxl_pci_probe_mode, .enable_device_hook = cxl_pci_enable_device_hook, - .disable_device = _cxl_pci_disable_device, - .release_device = _cxl_pci_disable_device, + .disable_device = cxl_pci_disable_device, + .release_device = cxl_pci_disable_device, .window_alignment = cxl_pci_window_alignment, .reset_secondary_bus = cxl_pci_reset_secondary_bus, .setup_msi_irqs = cxl_setup_msi_irqs, @@ -284,18 +307,13 @@ void cxl_pci_vphb_remove(struct cxl_afu *afu) */ } -static bool _cxl_pci_is_vphb_device(struct pci_controller *phb) -{ - return (phb->ops == &cxl_pcie_pci_ops); -} - bool cxl_pci_is_vphb_device(struct pci_dev *dev) { struct pci_controller *phb; phb = pci_bus_to_host(dev->bus); - return _cxl_pci_is_vphb_device(phb); + return (phb->ops == &cxl_pcie_pci_ops); } struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev) @@ -304,13 +322,7 @@ struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev) phb = pci_bus_to_host(dev->bus); - if (_cxl_pci_is_vphb_device(phb)) - return (struct cxl_afu *)phb->private_data; - - if (pnv_pci_on_cxl_phb(dev)) - return pnv_cxl_phb_to_afu(phb); - - return ERR_PTR(-ENODEV); + return (struct cxl_afu *)phb->private_data; } EXPORT_SYMBOL_GPL(cxl_pci_to_afu); diff --git a/drivers/misc/ocxl/context.c b/drivers/misc/ocxl/context.c index 95f74623113e..c10a940e3b38 100644 --- a/drivers/misc/ocxl/context.c +++ b/drivers/misc/ocxl/context.c @@ -86,7 +86,7 @@ out: return rc; } -static int map_afu_irq(struct vm_area_struct *vma, unsigned long address, +static vm_fault_t map_afu_irq(struct vm_area_struct *vma, unsigned long address, u64 offset, struct ocxl_context *ctx) { u64 trigger_addr; @@ -95,15 +95,15 @@ static int map_afu_irq(struct vm_area_struct *vma, unsigned long address, if (!trigger_addr) return VM_FAULT_SIGBUS; - vm_insert_pfn(vma, address, trigger_addr >> PAGE_SHIFT); - return VM_FAULT_NOPAGE; + return vmf_insert_pfn(vma, address, trigger_addr >> PAGE_SHIFT); } -static int map_pp_mmio(struct vm_area_struct *vma, unsigned long address, +static vm_fault_t map_pp_mmio(struct vm_area_struct *vma, unsigned long address, u64 offset, struct ocxl_context *ctx) { u64 pp_mmio_addr; int pasid_off; + vm_fault_t ret; if (offset >= ctx->afu->config.pp_mmio_stride) return VM_FAULT_SIGBUS; @@ -121,27 +121,27 @@ static int map_pp_mmio(struct vm_area_struct *vma, unsigned long address, pasid_off * ctx->afu->config.pp_mmio_stride + offset; - vm_insert_pfn(vma, address, pp_mmio_addr >> PAGE_SHIFT); + ret = vmf_insert_pfn(vma, address, pp_mmio_addr >> PAGE_SHIFT); mutex_unlock(&ctx->status_mutex); - return VM_FAULT_NOPAGE; + return ret; } -static int ocxl_mmap_fault(struct vm_fault *vmf) +static vm_fault_t ocxl_mmap_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct ocxl_context *ctx = vma->vm_file->private_data; u64 offset; - int rc; + vm_fault_t ret; offset = vmf->pgoff << PAGE_SHIFT; pr_debug("%s: pasid %d address 0x%lx offset 0x%llx\n", __func__, ctx->pasid, vmf->address, offset); if (offset < ctx->afu->irq_base_offset) - rc = map_pp_mmio(vma, vmf->address, offset, ctx); + ret = map_pp_mmio(vma, vmf->address, offset, ctx); else - rc = map_afu_irq(vma, vmf->address, offset, ctx); - return rc; + ret = map_afu_irq(vma, vmf->address, offset, ctx); + return ret; } static const struct vm_operations_struct ocxl_vmops = { diff --git a/drivers/misc/ocxl/link.c b/drivers/misc/ocxl/link.c index 88876ae8f330..a963b0a4a3c5 100644 --- a/drivers/misc/ocxl/link.c +++ b/drivers/misc/ocxl/link.c @@ -136,7 +136,7 @@ static void xsl_fault_handler_bh(struct work_struct *fault_work) int rc; /* - * We need to release a reference on the mm whenever exiting this + * We must release a reference on mm_users whenever exiting this * function (taken in the memory fault interrupt handler) */ rc = copro_handle_mm_fault(fault->pe_data.mm, fault->dar, fault->dsisr, @@ -172,7 +172,7 @@ static void xsl_fault_handler_bh(struct work_struct *fault_work) } r = RESTART; ack: - mmdrop(fault->pe_data.mm); + mmput(fault->pe_data.mm); ack_irq(spa, r); } @@ -184,6 +184,7 @@ static irqreturn_t xsl_fault_handler(int irq, void *data) struct pe_data *pe_data; struct ocxl_process_element *pe; int lpid, pid, tid; + bool schedule = false; read_irq(spa, &dsisr, &dar, &pe_handle); trace_ocxl_fault(spa->spa_mem, pe_handle, dsisr, dar, -1); @@ -226,14 +227,19 @@ static irqreturn_t xsl_fault_handler(int irq, void *data) } WARN_ON(pe_data->mm->context.id != pid); - spa->xsl_fault.pe = pe_handle; - spa->xsl_fault.dar = dar; - spa->xsl_fault.dsisr = dsisr; - spa->xsl_fault.pe_data = *pe_data; - mmgrab(pe_data->mm); /* mm count is released by bottom half */ - + if (mmget_not_zero(pe_data->mm)) { + spa->xsl_fault.pe = pe_handle; + spa->xsl_fault.dar = dar; + spa->xsl_fault.dsisr = dsisr; + spa->xsl_fault.pe_data = *pe_data; + schedule = true; + /* mm_users count released by bottom half */ + } rcu_read_unlock(); - schedule_work(&spa->xsl_fault.fault_work); + if (schedule) + schedule_work(&spa->xsl_fault.fault_work); + else + ack_irq(spa, ADDRESS_ERROR); return IRQ_HANDLED; } diff --git a/drivers/misc/ocxl/sysfs.c b/drivers/misc/ocxl/sysfs.c index d9753a1db14b..0ab1fd1b2682 100644 --- a/drivers/misc/ocxl/sysfs.c +++ b/drivers/misc/ocxl/sysfs.c @@ -64,7 +64,7 @@ static ssize_t global_mmio_read(struct file *filp, struct kobject *kobj, return count; } -static int global_mmio_fault(struct vm_fault *vmf) +static vm_fault_t global_mmio_fault(struct vm_fault *vmf) { struct vm_area_struct *vma = vmf->vma; struct ocxl_afu *afu = vma->vm_private_data; @@ -75,8 +75,7 @@ static int global_mmio_fault(struct vm_fault *vmf) offset = vmf->pgoff; offset += (afu->global_mmio_start >> PAGE_SHIFT); - vm_insert_pfn(vma, vmf->address, offset); - return VM_FAULT_NOPAGE; + return vmf_insert_pfn(vma, vmf->address, offset); } static const struct vm_operations_struct global_mmio_vmops = { diff --git a/drivers/tty/hvc/hvc_console.c b/drivers/tty/hvc/hvc_console.c index 7709fcc707f4..5414c4a87bea 100644 --- a/drivers/tty/hvc/hvc_console.c +++ b/drivers/tty/hvc/hvc_console.c @@ -73,7 +73,7 @@ static LIST_HEAD(hvc_structs); * Protect the list of hvc_struct instances from inserts and removals during * list traversal. */ -static DEFINE_SPINLOCK(hvc_structs_lock); +static DEFINE_MUTEX(hvc_structs_mutex); /* * This value is used to assign a tty->index value to a hvc_struct based @@ -83,7 +83,7 @@ static DEFINE_SPINLOCK(hvc_structs_lock); static int last_hvc = -1; /* - * Do not call this function with either the hvc_structs_lock or the hvc_struct + * Do not call this function with either the hvc_structs_mutex or the hvc_struct * lock held. If successful, this function increments the kref reference * count against the target hvc_struct so it should be released when finished. */ @@ -92,24 +92,46 @@ static struct hvc_struct *hvc_get_by_index(int index) struct hvc_struct *hp; unsigned long flags; - spin_lock(&hvc_structs_lock); + mutex_lock(&hvc_structs_mutex); list_for_each_entry(hp, &hvc_structs, next) { spin_lock_irqsave(&hp->lock, flags); if (hp->index == index) { tty_port_get(&hp->port); spin_unlock_irqrestore(&hp->lock, flags); - spin_unlock(&hvc_structs_lock); + mutex_unlock(&hvc_structs_mutex); return hp; } spin_unlock_irqrestore(&hp->lock, flags); } hp = NULL; + mutex_unlock(&hvc_structs_mutex); - spin_unlock(&hvc_structs_lock); return hp; } +static int __hvc_flush(const struct hv_ops *ops, uint32_t vtermno, bool wait) +{ + if (wait) + might_sleep(); + + if (ops->flush) + return ops->flush(vtermno, wait); + return 0; +} + +static int hvc_console_flush(const struct hv_ops *ops, uint32_t vtermno) +{ + return __hvc_flush(ops, vtermno, false); +} + +/* + * Wait for the console to flush before writing more to it. This sleeps. + */ +static int hvc_flush(struct hvc_struct *hp) +{ + return __hvc_flush(hp->ops, hp->vtermno, true); +} /* * Initial console vtermnos for console API usage prior to full console @@ -156,8 +178,12 @@ static void hvc_console_print(struct console *co, const char *b, if (r <= 0) { /* throw away characters on error * but spin in case of -EAGAIN */ - if (r != -EAGAIN) + if (r != -EAGAIN) { i = 0; + } else { + hvc_console_flush(cons_ops[index], + vtermnos[index]); + } } else if (r > 0) { i -= r; if (i > 0) @@ -165,6 +191,7 @@ static void hvc_console_print(struct console *co, const char *b, } } } + hvc_console_flush(cons_ops[index], vtermnos[index]); } static struct tty_driver *hvc_console_device(struct console *c, int *index) @@ -224,13 +251,13 @@ static void hvc_port_destruct(struct tty_port *port) struct hvc_struct *hp = container_of(port, struct hvc_struct, port); unsigned long flags; - spin_lock(&hvc_structs_lock); + mutex_lock(&hvc_structs_mutex); spin_lock_irqsave(&hp->lock, flags); list_del(&(hp->next)); spin_unlock_irqrestore(&hp->lock, flags); - spin_unlock(&hvc_structs_lock); + mutex_unlock(&hvc_structs_mutex); kfree(hp); } @@ -494,23 +521,32 @@ static int hvc_write(struct tty_struct *tty, const unsigned char *buf, int count if (hp->port.count <= 0) return -EIO; - spin_lock_irqsave(&hp->lock, flags); + while (count > 0) { + spin_lock_irqsave(&hp->lock, flags); - /* Push pending writes */ - if (hp->n_outbuf > 0) - hvc_push(hp); - - while (count > 0 && (rsize = hp->outbuf_size - hp->n_outbuf) > 0) { - if (rsize > count) - rsize = count; - memcpy(hp->outbuf + hp->n_outbuf, buf, rsize); - count -= rsize; - buf += rsize; - hp->n_outbuf += rsize; - written += rsize; - hvc_push(hp); + rsize = hp->outbuf_size - hp->n_outbuf; + + if (rsize) { + if (rsize > count) + rsize = count; + memcpy(hp->outbuf + hp->n_outbuf, buf, rsize); + count -= rsize; + buf += rsize; + hp->n_outbuf += rsize; + written += rsize; + } + + if (hp->n_outbuf > 0) + hvc_push(hp); + + spin_unlock_irqrestore(&hp->lock, flags); + + if (count) { + if (hp->n_outbuf > 0) + hvc_flush(hp); + cond_resched(); + } } - spin_unlock_irqrestore(&hp->lock, flags); /* * Racy, but harmless, kick thread if there is still pending data. @@ -590,10 +626,10 @@ static u32 timeout = MIN_TIMEOUT; #define HVC_POLL_READ 0x00000001 #define HVC_POLL_WRITE 0x00000002 -int hvc_poll(struct hvc_struct *hp) +static int __hvc_poll(struct hvc_struct *hp, bool may_sleep) { struct tty_struct *tty; - int i, n, poll_mask = 0; + int i, n, count, poll_mask = 0; char buf[N_INBUF] __ALIGNED__; unsigned long flags; int read_total = 0; @@ -612,6 +648,12 @@ int hvc_poll(struct hvc_struct *hp) timeout = (written_total) ? 0 : MIN_TIMEOUT; } + if (may_sleep) { + spin_unlock_irqrestore(&hp->lock, flags); + cond_resched(); + spin_lock_irqsave(&hp->lock, flags); + } + /* No tty attached, just skip */ tty = tty_port_tty_get(&hp->port); if (tty == NULL) @@ -619,7 +661,7 @@ int hvc_poll(struct hvc_struct *hp) /* Now check if we can get data (are we throttled ?) */ if (tty_throttled(tty)) - goto throttled; + goto out; /* If we aren't notifier driven and aren't throttled, we always * request a reschedule @@ -628,56 +670,58 @@ int hvc_poll(struct hvc_struct *hp) poll_mask |= HVC_POLL_READ; /* Read data if any */ - for (;;) { - int count = tty_buffer_request_room(&hp->port, N_INBUF); - /* If flip is full, just reschedule a later read */ - if (count == 0) { + count = tty_buffer_request_room(&hp->port, N_INBUF); + + /* If flip is full, just reschedule a later read */ + if (count == 0) { + poll_mask |= HVC_POLL_READ; + goto out; + } + + n = hp->ops->get_chars(hp->vtermno, buf, count); + if (n <= 0) { + /* Hangup the tty when disconnected from host */ + if (n == -EPIPE) { + spin_unlock_irqrestore(&hp->lock, flags); + tty_hangup(tty); + spin_lock_irqsave(&hp->lock, flags); + } else if ( n == -EAGAIN ) { + /* + * Some back-ends can only ensure a certain min + * num of bytes read, which may be > 'count'. + * Let the tty clear the flip buff to make room. + */ poll_mask |= HVC_POLL_READ; - break; } + goto out; + } - n = hp->ops->get_chars(hp->vtermno, buf, count); - if (n <= 0) { - /* Hangup the tty when disconnected from host */ - if (n == -EPIPE) { - spin_unlock_irqrestore(&hp->lock, flags); - tty_hangup(tty); - spin_lock_irqsave(&hp->lock, flags); - } else if ( n == -EAGAIN ) { - /* - * Some back-ends can only ensure a certain min - * num of bytes read, which may be > 'count'. - * Let the tty clear the flip buff to make room. - */ - poll_mask |= HVC_POLL_READ; - } - break; - } - for (i = 0; i < n; ++i) { + for (i = 0; i < n; ++i) { #ifdef CONFIG_MAGIC_SYSRQ - if (hp->index == hvc_console.index) { - /* Handle the SysRq Hack */ - /* XXX should support a sequence */ - if (buf[i] == '\x0f') { /* ^O */ - /* if ^O is pressed again, reset - * sysrq_pressed and flip ^O char */ - sysrq_pressed = !sysrq_pressed; - if (sysrq_pressed) - continue; - } else if (sysrq_pressed) { - handle_sysrq(buf[i]); - sysrq_pressed = 0; + if (hp->index == hvc_console.index) { + /* Handle the SysRq Hack */ + /* XXX should support a sequence */ + if (buf[i] == '\x0f') { /* ^O */ + /* if ^O is pressed again, reset + * sysrq_pressed and flip ^O char */ + sysrq_pressed = !sysrq_pressed; + if (sysrq_pressed) continue; - } + } else if (sysrq_pressed) { + handle_sysrq(buf[i]); + sysrq_pressed = 0; + continue; } -#endif /* CONFIG_MAGIC_SYSRQ */ - tty_insert_flip_char(&hp->port, buf[i], 0); } - - read_total += n; +#endif /* CONFIG_MAGIC_SYSRQ */ + tty_insert_flip_char(&hp->port, buf[i], 0); } - throttled: + if (n == count) + poll_mask |= HVC_POLL_READ; + read_total = n; + + out: /* Wakeup write queue if necessary */ if (hp->do_wakeup) { hp->do_wakeup = 0; @@ -697,6 +741,11 @@ int hvc_poll(struct hvc_struct *hp) return poll_mask; } + +int hvc_poll(struct hvc_struct *hp) +{ + return __hvc_poll(hp, false); +} EXPORT_SYMBOL_GPL(hvc_poll); /** @@ -733,11 +782,12 @@ static int khvcd(void *unused) try_to_freeze(); wmb(); if (!cpus_are_in_xmon()) { - spin_lock(&hvc_structs_lock); + mutex_lock(&hvc_structs_mutex); list_for_each_entry(hp, &hvc_structs, next) { - poll_mask |= hvc_poll(hp); + poll_mask |= __hvc_poll(hp, true); + cond_resched(); } - spin_unlock(&hvc_structs_lock); + mutex_unlock(&hvc_structs_mutex); } else poll_mask |= HVC_POLL_READ; if (hvc_kicked) @@ -871,7 +921,7 @@ struct hvc_struct *hvc_alloc(uint32_t vtermno, int data, INIT_WORK(&hp->tty_resize, hvc_set_winsz); spin_lock_init(&hp->lock); - spin_lock(&hvc_structs_lock); + mutex_lock(&hvc_structs_mutex); /* * find index to use: @@ -891,7 +941,7 @@ struct hvc_struct *hvc_alloc(uint32_t vtermno, int data, vtermnos[i] = vtermno; list_add_tail(&(hp->next), &hvc_structs); - spin_unlock(&hvc_structs_lock); + mutex_unlock(&hvc_structs_mutex); /* check if we need to re-register the kernel console */ hvc_check_console(i); diff --git a/drivers/tty/hvc/hvc_console.h b/drivers/tty/hvc/hvc_console.h index ea63090e013f..e9319954c832 100644 --- a/drivers/tty/hvc/hvc_console.h +++ b/drivers/tty/hvc/hvc_console.h @@ -54,6 +54,7 @@ struct hvc_struct { struct hv_ops { int (*get_chars)(uint32_t vtermno, char *buf, int count); int (*put_chars)(uint32_t vtermno, const char *buf, int count); + int (*flush)(uint32_t vtermno, bool wait); /* Callbacks for notification. Called in open, close and hangup */ int (*notifier_add)(struct hvc_struct *hp, int irq); diff --git a/drivers/tty/hvc/hvc_opal.c b/drivers/tty/hvc/hvc_opal.c index 9645c0062a90..f631f8bee308 100644 --- a/drivers/tty/hvc/hvc_opal.c +++ b/drivers/tty/hvc/hvc_opal.c @@ -183,9 +183,15 @@ static int hvc_opal_probe(struct platform_device *dev) return -ENOMEM; pv->proto = proto; hvc_opal_privs[termno] = pv; - if (proto == HV_PROTOCOL_HVSI) - hvsilib_init(&pv->hvsi, opal_get_chars, opal_put_chars, + if (proto == HV_PROTOCOL_HVSI) { + /* + * We want put_chars to be atomic to avoid mangling of + * hvsi packets. + */ + hvsilib_init(&pv->hvsi, + opal_get_chars, opal_put_chars_atomic, termno, 0); + } /* Instanciate now to establish a mapping index==vtermno */ hvc_instantiate(termno, termno, ops); @@ -275,6 +281,11 @@ static void udbg_opal_putc(char c) count = hvc_opal_hvsi_put_chars(termno, &c, 1); break; } + + /* This is needed for the cosole to flush + * when there aren't any interrupts. + */ + opal_flush_console(termno); } while(count == 0 || count == -EAGAIN); } @@ -302,14 +313,8 @@ static int udbg_opal_getc(void) int ch; for (;;) { ch = udbg_opal_getc_poll(); - if (ch == -1) { - /* This shouldn't be needed...but... */ - volatile unsigned long delay; - for (delay=0; delay < 2000000; delay++) - ; - } else { + if (ch != -1) return ch; - } } } @@ -370,8 +375,9 @@ void __init hvc_opal_init_early(void) else if (of_device_is_compatible(stdout_node,"ibm,opal-console-hvsi")) { hvc_opal_boot_priv.proto = HV_PROTOCOL_HVSI; ops = &hvc_opal_hvsi_ops; - hvsilib_init(&hvc_opal_boot_priv.hvsi, opal_get_chars, - opal_put_chars, index, 1); + hvsilib_init(&hvc_opal_boot_priv.hvsi, + opal_get_chars, opal_put_chars_atomic, + index, 1); /* HVSI, perform the handshake now */ hvsilib_establish(&hvc_opal_boot_priv.hvsi); pr_devel("hvc_opal: Found HVSI console\n"); @@ -403,7 +409,8 @@ void __init udbg_init_debug_opal_hvsi(void) hvc_opal_privs[index] = &hvc_opal_boot_priv; hvc_opal_boot_termno = index; udbg_init_opal_common(); - hvsilib_init(&hvc_opal_boot_priv.hvsi, opal_get_chars, opal_put_chars, + hvsilib_init(&hvc_opal_boot_priv.hvsi, + opal_get_chars, opal_put_chars_atomic, index, 1); hvsilib_establish(&hvc_opal_boot_priv.hvsi); } diff --git a/drivers/usb/host/ehci-ps3.c b/drivers/usb/host/ehci-ps3.c index 8c733492d8fe..454d8c624a3f 100644 --- a/drivers/usb/host/ehci-ps3.c +++ b/drivers/usb/host/ehci-ps3.c @@ -86,7 +86,7 @@ static int ps3_ehci_probe(struct ps3_system_bus_device *dev) int result; struct usb_hcd *hcd; unsigned int virq; - static u64 dummy_mask = DMA_BIT_MASK(32); + static u64 dummy_mask; if (usb_disabled()) { result = -ENODEV; @@ -131,7 +131,9 @@ static int ps3_ehci_probe(struct ps3_system_bus_device *dev) goto fail_irq; } - dev->core.dma_mask = &dummy_mask; /* FIXME: for improper usb code */ + dummy_mask = DMA_BIT_MASK(32); + dev->core.dma_mask = &dummy_mask; + dma_set_coherent_mask(&dev->core, dummy_mask); hcd = usb_create_hcd(&ps3_ehci_hc_driver, &dev->core, dev_name(&dev->core)); diff --git a/drivers/usb/host/ohci-ps3.c b/drivers/usb/host/ohci-ps3.c index 20a23d795adf..395f9d3bc849 100644 --- a/drivers/usb/host/ohci-ps3.c +++ b/drivers/usb/host/ohci-ps3.c @@ -69,7 +69,7 @@ static int ps3_ohci_probe(struct ps3_system_bus_device *dev) int result; struct usb_hcd *hcd; unsigned int virq; - static u64 dummy_mask = DMA_BIT_MASK(32); + static u64 dummy_mask; if (usb_disabled()) { result = -ENODEV; @@ -115,7 +115,9 @@ static int ps3_ohci_probe(struct ps3_system_bus_device *dev) goto fail_irq; } - dev->core.dma_mask = &dummy_mask; /* FIXME: for improper usb code */ + dummy_mask = DMA_BIT_MASK(32); + dev->core.dma_mask = &dummy_mask; + dma_set_coherent_mask(&dev->core, dummy_mask); hcd = usb_create_hcd(&ps3_ohci_hc_driver, &dev->core, dev_name(&dev->core)); diff --git a/drivers/vfio/vfio_iommu_spapr_tce.c b/drivers/vfio/vfio_iommu_spapr_tce.c index 7cd63b0c1a46..96721b154454 100644 --- a/drivers/vfio/vfio_iommu_spapr_tce.c +++ b/drivers/vfio/vfio_iommu_spapr_tce.c @@ -211,44 +211,6 @@ static long tce_iommu_register_pages(struct tce_container *container, return 0; } -static long tce_iommu_userspace_view_alloc(struct iommu_table *tbl, - struct mm_struct *mm) -{ - unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) * - tbl->it_size, PAGE_SIZE); - unsigned long *uas; - long ret; - - BUG_ON(tbl->it_userspace); - - ret = try_increment_locked_vm(mm, cb >> PAGE_SHIFT); - if (ret) - return ret; - - uas = vzalloc(cb); - if (!uas) { - decrement_locked_vm(mm, cb >> PAGE_SHIFT); - return -ENOMEM; - } - tbl->it_userspace = uas; - - return 0; -} - -static void tce_iommu_userspace_view_free(struct iommu_table *tbl, - struct mm_struct *mm) -{ - unsigned long cb = _ALIGN_UP(sizeof(tbl->it_userspace[0]) * - tbl->it_size, PAGE_SIZE); - - if (!tbl->it_userspace) - return; - - vfree(tbl->it_userspace); - tbl->it_userspace = NULL; - decrement_locked_vm(mm, cb >> PAGE_SHIFT); -} - static bool tce_page_is_contained(struct page *page, unsigned page_shift) { /* @@ -482,20 +444,20 @@ static void tce_iommu_unuse_page_v2(struct tce_container *container, struct mm_iommu_table_group_mem_t *mem = NULL; int ret; unsigned long hpa = 0; - unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); + __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry); if (!pua) return; - ret = tce_iommu_prereg_ua_to_hpa(container, *pua, tbl->it_page_shift, - &hpa, &mem); + ret = tce_iommu_prereg_ua_to_hpa(container, be64_to_cpu(*pua), + tbl->it_page_shift, &hpa, &mem); if (ret) - pr_debug("%s: tce %lx at #%lx was not cached, ret=%d\n", - __func__, *pua, entry, ret); + pr_debug("%s: tce %llx at #%lx was not cached, ret=%d\n", + __func__, be64_to_cpu(*pua), entry, ret); if (mem) mm_iommu_mapped_dec(mem); - *pua = 0; + *pua = cpu_to_be64(0); } static int tce_iommu_clear(struct tce_container *container, @@ -599,16 +561,9 @@ static long tce_iommu_build_v2(struct tce_container *container, unsigned long hpa; enum dma_data_direction dirtmp; - if (!tbl->it_userspace) { - ret = tce_iommu_userspace_view_alloc(tbl, container->mm); - if (ret) - return ret; - } - for (i = 0; i < pages; ++i) { struct mm_iommu_table_group_mem_t *mem = NULL; - unsigned long *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, - entry + i); + __be64 *pua = IOMMU_TABLE_USERSPACE_ENTRY(tbl, entry + i); ret = tce_iommu_prereg_ua_to_hpa(container, tce, tbl->it_page_shift, &hpa, &mem); @@ -642,7 +597,7 @@ static long tce_iommu_build_v2(struct tce_container *container, if (dirtmp != DMA_NONE) tce_iommu_unuse_page_v2(container, tbl, entry + i); - *pua = tce; + *pua = cpu_to_be64(tce); tce += IOMMU_PAGE_SIZE(tbl); } @@ -676,7 +631,7 @@ static long tce_iommu_create_table(struct tce_container *container, page_shift, window_size, levels, ptbl); WARN_ON(!ret && !(*ptbl)->it_ops->free); - WARN_ON(!ret && ((*ptbl)->it_allocated_size != table_size)); + WARN_ON(!ret && ((*ptbl)->it_allocated_size > table_size)); return ret; } @@ -686,7 +641,6 @@ static void tce_iommu_free_table(struct tce_container *container, { unsigned long pages = tbl->it_allocated_size >> PAGE_SHIFT; - tce_iommu_userspace_view_free(tbl, container->mm); iommu_tce_table_put(tbl); decrement_locked_vm(container->mm, pages); } @@ -1201,7 +1155,6 @@ static void tce_iommu_release_ownership(struct tce_container *container, continue; tce_iommu_clear(container, tbl, tbl->it_offset, tbl->it_size); - tce_iommu_userspace_view_free(tbl, container->mm); if (tbl->it_map) iommu_release_ownership(tbl); diff --git a/fs/ext2/inode.c b/fs/ext2/inode.c index 71635909df3b..7f7ee18fe179 100644 --- a/fs/ext2/inode.c +++ b/fs/ext2/inode.c @@ -86,7 +86,7 @@ void ext2_evict_inode(struct inode * inode) if (want_delete) { sb_start_intwrite(inode->i_sb); /* set dtime */ - EXT2_I(inode)->i_dtime = get_seconds(); + EXT2_I(inode)->i_dtime = ktime_get_real_seconds(); mark_inode_dirty(inode); __ext2_write_inode(inode, inode_needs_sync(inode)); /* truncate to 0 */ diff --git a/fs/ext2/super.c b/fs/ext2/super.c index 8ff53f8da3bc..73bd58fa13de 100644 --- a/fs/ext2/super.c +++ b/fs/ext2/super.c @@ -682,7 +682,8 @@ static int ext2_setup_super (struct super_block * sb, "running e2fsck is recommended"); else if (le32_to_cpu(es->s_checkinterval) && (le32_to_cpu(es->s_lastcheck) + - le32_to_cpu(es->s_checkinterval) <= get_seconds())) + le32_to_cpu(es->s_checkinterval) <= + ktime_get_real_seconds())) ext2_msg(sb, KERN_WARNING, "warning: checktime reached, " "running e2fsck is recommended"); @@ -1248,7 +1249,7 @@ void ext2_sync_super(struct super_block *sb, struct ext2_super_block *es, spin_lock(&EXT2_SB(sb)->s_lock); es->s_free_blocks_count = cpu_to_le32(ext2_count_free_blocks(sb)); es->s_free_inodes_count = cpu_to_le32(ext2_count_free_inodes(sb)); - es->s_wtime = cpu_to_le32(get_seconds()); + es->s_wtime = cpu_to_le32(ktime_get_real_seconds()); /* unlock before we do IO */ spin_unlock(&EXT2_SB(sb)->s_lock); mark_buffer_dirty(EXT2_SB(sb)->s_sbh); @@ -1360,7 +1361,7 @@ static int ext2_remount (struct super_block * sb, int * flags, char * data) * the rdonly flag and then mark the partition as valid again. */ es->s_state = cpu_to_le16(sbi->s_mount_state); - es->s_mtime = cpu_to_le32(get_seconds()); + es->s_mtime = cpu_to_le32(ktime_get_real_seconds()); spin_unlock(&sbi->s_lock); err = dquot_suspend(sb, -1); diff --git a/fs/notify/fanotify/fanotify_user.c b/fs/notify/fanotify/fanotify_user.c index ec4d8c59d0e3..d736a833fe39 100644 --- a/fs/notify/fanotify/fanotify_user.c +++ b/fs/notify/fanotify/fanotify_user.c @@ -524,17 +524,16 @@ static __u32 fanotify_mark_remove_from_mask(struct fsnotify_mark *fsn_mark, return mask & oldmask; } -static int fanotify_remove_vfsmount_mark(struct fsnotify_group *group, - struct vfsmount *mnt, __u32 mask, - unsigned int flags) +static int fanotify_remove_mark(struct fsnotify_group *group, + fsnotify_connp_t *connp, __u32 mask, + unsigned int flags) { struct fsnotify_mark *fsn_mark = NULL; __u32 removed; int destroy_mark; mutex_lock(&group->mark_mutex); - fsn_mark = fsnotify_find_mark(&real_mount(mnt)->mnt_fsnotify_marks, - group); + fsn_mark = fsnotify_find_mark(connp, group); if (!fsn_mark) { mutex_unlock(&group->mark_mutex); return -ENOENT; @@ -542,47 +541,33 @@ static int fanotify_remove_vfsmount_mark(struct fsnotify_group *group, removed = fanotify_mark_remove_from_mask(fsn_mark, mask, flags, &destroy_mark); - if (removed & real_mount(mnt)->mnt_fsnotify_mask) - fsnotify_recalc_mask(real_mount(mnt)->mnt_fsnotify_marks); + if (removed & fsnotify_conn_mask(fsn_mark->connector)) + fsnotify_recalc_mask(fsn_mark->connector); if (destroy_mark) fsnotify_detach_mark(fsn_mark); mutex_unlock(&group->mark_mutex); if (destroy_mark) fsnotify_free_mark(fsn_mark); + /* matches the fsnotify_find_mark() */ fsnotify_put_mark(fsn_mark); return 0; } +static int fanotify_remove_vfsmount_mark(struct fsnotify_group *group, + struct vfsmount *mnt, __u32 mask, + unsigned int flags) +{ + return fanotify_remove_mark(group, &real_mount(mnt)->mnt_fsnotify_marks, + mask, flags); +} + static int fanotify_remove_inode_mark(struct fsnotify_group *group, struct inode *inode, __u32 mask, unsigned int flags) { - struct fsnotify_mark *fsn_mark = NULL; - __u32 removed; - int destroy_mark; - - mutex_lock(&group->mark_mutex); - fsn_mark = fsnotify_find_mark(&inode->i_fsnotify_marks, group); - if (!fsn_mark) { - mutex_unlock(&group->mark_mutex); - return -ENOENT; - } - - removed = fanotify_mark_remove_from_mask(fsn_mark, mask, flags, - &destroy_mark); - if (removed & inode->i_fsnotify_mask) - fsnotify_recalc_mask(inode->i_fsnotify_marks); - if (destroy_mark) - fsnotify_detach_mark(fsn_mark); - mutex_unlock(&group->mark_mutex); - if (destroy_mark) - fsnotify_free_mark(fsn_mark); - - /* matches the fsnotify_find_mark() */ - fsnotify_put_mark(fsn_mark); - - return 0; + return fanotify_remove_mark(group, &inode->i_fsnotify_marks, mask, + flags); } static __u32 fanotify_mark_add_to_mask(struct fsnotify_mark *fsn_mark, @@ -615,8 +600,8 @@ static __u32 fanotify_mark_add_to_mask(struct fsnotify_mark *fsn_mark, } static struct fsnotify_mark *fanotify_add_new_mark(struct fsnotify_group *group, - struct inode *inode, - struct vfsmount *mnt) + fsnotify_connp_t *connp, + unsigned int type) { struct fsnotify_mark *mark; int ret; @@ -629,7 +614,7 @@ static struct fsnotify_mark *fanotify_add_new_mark(struct fsnotify_group *group, return ERR_PTR(-ENOMEM); fsnotify_init_mark(mark, group); - ret = fsnotify_add_mark_locked(mark, inode, mnt, 0); + ret = fsnotify_add_mark_locked(mark, connp, type, 0); if (ret) { fsnotify_put_mark(mark); return ERR_PTR(ret); @@ -639,39 +624,43 @@ static struct fsnotify_mark *fanotify_add_new_mark(struct fsnotify_group *group, } -static int fanotify_add_vfsmount_mark(struct fsnotify_group *group, - struct vfsmount *mnt, __u32 mask, - unsigned int flags) +static int fanotify_add_mark(struct fsnotify_group *group, + fsnotify_connp_t *connp, unsigned int type, + __u32 mask, unsigned int flags) { struct fsnotify_mark *fsn_mark; __u32 added; mutex_lock(&group->mark_mutex); - fsn_mark = fsnotify_find_mark(&real_mount(mnt)->mnt_fsnotify_marks, - group); + fsn_mark = fsnotify_find_mark(connp, group); if (!fsn_mark) { - fsn_mark = fanotify_add_new_mark(group, NULL, mnt); + fsn_mark = fanotify_add_new_mark(group, connp, type); if (IS_ERR(fsn_mark)) { mutex_unlock(&group->mark_mutex); return PTR_ERR(fsn_mark); } } added = fanotify_mark_add_to_mask(fsn_mark, mask, flags); - if (added & ~real_mount(mnt)->mnt_fsnotify_mask) - fsnotify_recalc_mask(real_mount(mnt)->mnt_fsnotify_marks); + if (added & ~fsnotify_conn_mask(fsn_mark->connector)) + fsnotify_recalc_mask(fsn_mark->connector); mutex_unlock(&group->mark_mutex); fsnotify_put_mark(fsn_mark); return 0; } +static int fanotify_add_vfsmount_mark(struct fsnotify_group *group, + struct vfsmount *mnt, __u32 mask, + unsigned int flags) +{ + return fanotify_add_mark(group, &real_mount(mnt)->mnt_fsnotify_marks, + FSNOTIFY_OBJ_TYPE_VFSMOUNT, mask, flags); +} + static int fanotify_add_inode_mark(struct fsnotify_group *group, struct inode *inode, __u32 mask, unsigned int flags) { - struct fsnotify_mark *fsn_mark; - __u32 added; - pr_debug("%s: group=%p inode=%p\n", __func__, group, inode); /* @@ -684,22 +673,8 @@ static int fanotify_add_inode_mark(struct fsnotify_group *group, (atomic_read(&inode->i_writecount) > 0)) return 0; - mutex_lock(&group->mark_mutex); - fsn_mark = fsnotify_find_mark(&inode->i_fsnotify_marks, group); - if (!fsn_mark) { - fsn_mark = fanotify_add_new_mark(group, inode, NULL); - if (IS_ERR(fsn_mark)) { - mutex_unlock(&group->mark_mutex); - return PTR_ERR(fsn_mark); - } - } - added = fanotify_mark_add_to_mask(fsn_mark, mask, flags); - if (added & ~inode->i_fsnotify_mask) - fsnotify_recalc_mask(inode->i_fsnotify_marks); - mutex_unlock(&group->mark_mutex); - - fsnotify_put_mark(fsn_mark); - return 0; + return fanotify_add_mark(group, &inode->i_fsnotify_marks, + FSNOTIFY_OBJ_TYPE_INODE, mask, flags); } /* fanotify syscalls */ diff --git a/fs/notify/fdinfo.c b/fs/notify/fdinfo.c index 10aac1942c9f..86fcf5814279 100644 --- a/fs/notify/fdinfo.c +++ b/fs/notify/fdinfo.c @@ -15,7 +15,7 @@ #include <linux/exportfs.h> #include "inotify/inotify.h" -#include "../fs/mount.h" +#include "fsnotify.h" #if defined(CONFIG_PROC_FS) @@ -81,7 +81,7 @@ static void inotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark) return; inode_mark = container_of(mark, struct inotify_inode_mark, fsn_mark); - inode = igrab(mark->connector->inode); + inode = igrab(fsnotify_conn_inode(mark->connector)); if (inode) { /* * IN_ALL_EVENTS represents all of the mask bits @@ -117,7 +117,7 @@ static void fanotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark) mflags |= FAN_MARK_IGNORED_SURV_MODIFY; if (mark->connector->type == FSNOTIFY_OBJ_TYPE_INODE) { - inode = igrab(mark->connector->inode); + inode = igrab(fsnotify_conn_inode(mark->connector)); if (!inode) return; seq_printf(m, "fanotify ino:%lx sdev:%x mflags:%x mask:%x ignored_mask:%x ", @@ -127,7 +127,7 @@ static void fanotify_fdinfo(struct seq_file *m, struct fsnotify_mark *mark) seq_putc(m, '\n'); iput(inode); } else if (mark->connector->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) { - struct mount *mnt = real_mount(mark->connector->mnt); + struct mount *mnt = fsnotify_conn_mount(mark->connector); seq_printf(m, "fanotify mnt_id:%x mflags:%x mask:%x ignored_mask:%x\n", mnt->mnt_id, mflags, mark->mask, mark->ignored_mask); diff --git a/fs/notify/fsnotify.h b/fs/notify/fsnotify.h index 34515d2c4ba3..7902653dd577 100644 --- a/fs/notify/fsnotify.h +++ b/fs/notify/fsnotify.h @@ -9,6 +9,18 @@ #include "../mount.h" +static inline struct inode *fsnotify_conn_inode( + struct fsnotify_mark_connector *conn) +{ + return container_of(conn->obj, struct inode, i_fsnotify_marks); +} + +static inline struct mount *fsnotify_conn_mount( + struct fsnotify_mark_connector *conn) +{ + return container_of(conn->obj, struct mount, mnt_fsnotify_marks); +} + /* destroy all events sitting in this groups notification queue */ extern void fsnotify_flush_notify(struct fsnotify_group *group); @@ -19,8 +31,8 @@ extern struct srcu_struct fsnotify_mark_srcu; extern int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b); -/* Destroy all marks connected via given connector */ -extern void fsnotify_destroy_marks(struct fsnotify_mark_connector __rcu **connp); +/* Destroy all marks attached to an object via connector */ +extern void fsnotify_destroy_marks(fsnotify_connp_t *connp); /* run the list of all marks associated with inode and destroy them */ static inline void fsnotify_clear_marks_by_inode(struct inode *inode) { diff --git a/fs/notify/inotify/inotify_user.c b/fs/notify/inotify/inotify_user.c index 1cf5b779d862..6f48d325c350 100644 --- a/fs/notify/inotify/inotify_user.c +++ b/fs/notify/inotify/inotify_user.c @@ -510,6 +510,7 @@ static int inotify_update_existing_watch(struct fsnotify_group *group, __u32 old_mask, new_mask; __u32 mask; int add = (arg & IN_MASK_ADD); + int create = (arg & IN_MASK_CREATE); int ret; mask = inotify_arg_to_mask(arg); @@ -517,6 +518,8 @@ static int inotify_update_existing_watch(struct fsnotify_group *group, fsn_mark = fsnotify_find_mark(&inode->i_fsnotify_marks, group); if (!fsn_mark) return -ENOENT; + else if (create) + return -EEXIST; i_mark = container_of(fsn_mark, struct inotify_inode_mark, fsn_mark); @@ -718,6 +721,10 @@ SYSCALL_DEFINE3(inotify_add_watch, int, fd, const char __user *, pathname, if (unlikely(!f.file)) return -EBADF; + /* IN_MASK_ADD and IN_MASK_CREATE don't make sense together */ + if (unlikely((mask & IN_MASK_ADD) && (mask & IN_MASK_CREATE))) + return -EINVAL; + /* verify that this is indeed an inotify instance */ if (unlikely(f.file->f_op != &inotify_fops)) { ret = -EINVAL; @@ -806,7 +813,7 @@ static int __init inotify_user_setup(void) BUILD_BUG_ON(IN_ISDIR != FS_ISDIR); BUILD_BUG_ON(IN_ONESHOT != FS_IN_ONESHOT); - BUG_ON(hweight32(ALL_INOTIFY_BITS) != 21); + BUG_ON(hweight32(ALL_INOTIFY_BITS) != 22); inotify_inode_mark_cachep = KMEM_CACHE(inotify_inode_mark, SLAB_PANIC); diff --git a/fs/notify/mark.c b/fs/notify/mark.c index 61f4c5fa34c7..05506d60131c 100644 --- a/fs/notify/mark.c +++ b/fs/notify/mark.c @@ -109,6 +109,23 @@ void fsnotify_get_mark(struct fsnotify_mark *mark) refcount_inc(&mark->refcnt); } +static __u32 *fsnotify_conn_mask_p(struct fsnotify_mark_connector *conn) +{ + if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) + return &fsnotify_conn_inode(conn)->i_fsnotify_mask; + else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) + return &fsnotify_conn_mount(conn)->mnt_fsnotify_mask; + return NULL; +} + +__u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn) +{ + if (WARN_ON(!fsnotify_valid_obj_type(conn->type))) + return 0; + + return *fsnotify_conn_mask_p(conn); +} + static void __fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) { u32 new_mask = 0; @@ -119,15 +136,15 @@ static void __fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) if (mark->flags & FSNOTIFY_MARK_FLAG_ATTACHED) new_mask |= mark->mask; } - if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) - conn->inode->i_fsnotify_mask = new_mask; - else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) - real_mount(conn->mnt)->mnt_fsnotify_mask = new_mask; + if (WARN_ON(!fsnotify_valid_obj_type(conn->type))) + return; + + *fsnotify_conn_mask_p(conn) = new_mask; } /* * Calculate mask of events for a list of marks. The caller must make sure - * connector and connector->inode cannot disappear under us. Callers achieve + * connector and connector->obj cannot disappear under us. Callers achieve * this by holding a mark->lock or mark->group->mark_mutex for a mark on this * list. */ @@ -140,7 +157,8 @@ void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn) __fsnotify_recalc_mask(conn); spin_unlock(&conn->lock); if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) - __fsnotify_update_child_dentry_flags(conn->inode); + __fsnotify_update_child_dentry_flags( + fsnotify_conn_inode(conn)); } /* Free all connectors queued for freeing once SRCU period ends */ @@ -166,20 +184,20 @@ static struct inode *fsnotify_detach_connector_from_object( { struct inode *inode = NULL; + if (conn->type == FSNOTIFY_OBJ_TYPE_DETACHED) + return NULL; + if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) { - inode = conn->inode; - rcu_assign_pointer(inode->i_fsnotify_marks, NULL); + inode = fsnotify_conn_inode(conn); inode->i_fsnotify_mask = 0; - conn->inode = NULL; - conn->type = FSNOTIFY_OBJ_TYPE_DETACHED; } else if (conn->type == FSNOTIFY_OBJ_TYPE_VFSMOUNT) { - rcu_assign_pointer(real_mount(conn->mnt)->mnt_fsnotify_marks, - NULL); - real_mount(conn->mnt)->mnt_fsnotify_mask = 0; - conn->mnt = NULL; - conn->type = FSNOTIFY_OBJ_TYPE_DETACHED; + fsnotify_conn_mount(conn)->mnt_fsnotify_mask = 0; } + rcu_assign_pointer(*(conn->obj), NULL); + conn->obj = NULL; + conn->type = FSNOTIFY_OBJ_TYPE_DETACHED; + return inode; } @@ -436,11 +454,10 @@ int fsnotify_compare_groups(struct fsnotify_group *a, struct fsnotify_group *b) return -1; } -static int fsnotify_attach_connector_to_object( - struct fsnotify_mark_connector __rcu **connp, - struct inode *inode, - struct vfsmount *mnt) +static int fsnotify_attach_connector_to_object(fsnotify_connp_t *connp, + unsigned int type) { + struct inode *inode = NULL; struct fsnotify_mark_connector *conn; conn = kmem_cache_alloc(fsnotify_mark_connector_cachep, GFP_KERNEL); @@ -448,13 +465,10 @@ static int fsnotify_attach_connector_to_object( return -ENOMEM; spin_lock_init(&conn->lock); INIT_HLIST_HEAD(&conn->list); - if (inode) { - conn->type = FSNOTIFY_OBJ_TYPE_INODE; - conn->inode = igrab(inode); - } else { - conn->type = FSNOTIFY_OBJ_TYPE_VFSMOUNT; - conn->mnt = mnt; - } + conn->type = type; + conn->obj = connp; + if (conn->type == FSNOTIFY_OBJ_TYPE_INODE) + inode = igrab(fsnotify_conn_inode(conn)); /* * cmpxchg() provides the barrier so that readers of *connp can see * only initialized structure @@ -476,7 +490,7 @@ static int fsnotify_attach_connector_to_object( * they are sure list cannot go away under them. */ static struct fsnotify_mark_connector *fsnotify_grab_connector( - struct fsnotify_mark_connector __rcu **connp) + fsnotify_connp_t *connp) { struct fsnotify_mark_connector *conn; int idx; @@ -503,27 +517,22 @@ out: * priority, highest number first, and then by the group's location in memory. */ static int fsnotify_add_mark_list(struct fsnotify_mark *mark, - struct inode *inode, struct vfsmount *mnt, + fsnotify_connp_t *connp, unsigned int type, int allow_dups) { struct fsnotify_mark *lmark, *last = NULL; struct fsnotify_mark_connector *conn; - struct fsnotify_mark_connector __rcu **connp; int cmp; int err = 0; - if (WARN_ON(!inode && !mnt)) + if (WARN_ON(!fsnotify_valid_obj_type(type))) return -EINVAL; - if (inode) - connp = &inode->i_fsnotify_marks; - else - connp = &real_mount(mnt)->mnt_fsnotify_marks; restart: spin_lock(&mark->lock); conn = fsnotify_grab_connector(connp); if (!conn) { spin_unlock(&mark->lock); - err = fsnotify_attach_connector_to_object(connp, inode, mnt); + err = fsnotify_attach_connector_to_object(connp, type); if (err) return err; goto restart; @@ -569,14 +578,13 @@ out_err: * These marks may be used for the fsnotify backend to determine which * event types should be delivered to which group. */ -int fsnotify_add_mark_locked(struct fsnotify_mark *mark, struct inode *inode, - struct vfsmount *mnt, int allow_dups) +int fsnotify_add_mark_locked(struct fsnotify_mark *mark, + fsnotify_connp_t *connp, unsigned int type, + int allow_dups) { struct fsnotify_group *group = mark->group; int ret = 0; - BUG_ON(inode && mnt); - BUG_ON(!inode && !mnt); BUG_ON(!mutex_is_locked(&group->mark_mutex)); /* @@ -593,7 +601,7 @@ int fsnotify_add_mark_locked(struct fsnotify_mark *mark, struct inode *inode, fsnotify_get_mark(mark); /* for g_list */ spin_unlock(&mark->lock); - ret = fsnotify_add_mark_list(mark, inode, mnt, allow_dups); + ret = fsnotify_add_mark_list(mark, connp, type, allow_dups); if (ret) goto err; @@ -613,14 +621,14 @@ err: return ret; } -int fsnotify_add_mark(struct fsnotify_mark *mark, struct inode *inode, - struct vfsmount *mnt, int allow_dups) +int fsnotify_add_mark(struct fsnotify_mark *mark, fsnotify_connp_t *connp, + unsigned int type, int allow_dups) { int ret; struct fsnotify_group *group = mark->group; mutex_lock(&group->mark_mutex); - ret = fsnotify_add_mark_locked(mark, inode, mnt, allow_dups); + ret = fsnotify_add_mark_locked(mark, connp, type, allow_dups); mutex_unlock(&group->mark_mutex); return ret; } @@ -629,9 +637,8 @@ int fsnotify_add_mark(struct fsnotify_mark *mark, struct inode *inode, * Given a list of marks, find the mark associated with given group. If found * take a reference to that mark and return it, else return NULL. */ -struct fsnotify_mark *fsnotify_find_mark( - struct fsnotify_mark_connector __rcu **connp, - struct fsnotify_group *group) +struct fsnotify_mark *fsnotify_find_mark(fsnotify_connp_t *connp, + struct fsnotify_group *group) { struct fsnotify_mark_connector *conn; struct fsnotify_mark *mark; @@ -697,8 +704,8 @@ clear: } } -/* Destroy all marks attached to inode / vfsmount */ -void fsnotify_destroy_marks(struct fsnotify_mark_connector __rcu **connp) +/* Destroy all marks attached to an object via connector */ +void fsnotify_destroy_marks(fsnotify_connp_t *connp) { struct fsnotify_mark_connector *conn; struct fsnotify_mark *mark, *old_mark = NULL; diff --git a/fs/udf/ialloc.c b/fs/udf/ialloc.c index 56569023783b..f8e5872f7cc2 100644 --- a/fs/udf/ialloc.c +++ b/fs/udf/ialloc.c @@ -125,7 +125,7 @@ struct inode *udf_new_inode(struct inode *dir, umode_t mode) else iinfo->i_alloc_type = ICBTAG_FLAG_AD_LONG; inode->i_mtime = inode->i_atime = inode->i_ctime = current_time(inode); - iinfo->i_crtime = timespec64_to_timespec(inode->i_mtime); + iinfo->i_crtime = inode->i_mtime; if (unlikely(insert_inode_locked(inode) < 0)) { make_bad_inode(inode); iput(inode); diff --git a/fs/udf/inode.c b/fs/udf/inode.c index 9915a58fbabd..5df554a9f9c9 100644 --- a/fs/udf/inode.c +++ b/fs/udf/inode.c @@ -1270,7 +1270,6 @@ static int udf_read_inode(struct inode *inode, bool hidden_inode) struct udf_inode_info *iinfo = UDF_I(inode); struct udf_sb_info *sbi = UDF_SB(inode->i_sb); struct kernel_lb_addr *iloc = &iinfo->i_location; - struct timespec ts; unsigned int link_count; unsigned int indirections = 0; int bs = inode->i_sb->s_blocksize; @@ -1443,12 +1442,9 @@ reread: inode->i_blocks = le64_to_cpu(fe->logicalBlocksRecorded) << (inode->i_sb->s_blocksize_bits - 9); - udf_disk_stamp_to_time(&ts, fe->accessTime); - inode->i_atime = timespec_to_timespec64(ts); - udf_disk_stamp_to_time(&ts, fe->modificationTime); - inode->i_mtime = timespec_to_timespec64(ts); - udf_disk_stamp_to_time(&ts, fe->attrTime); - inode->i_ctime = timespec_to_timespec64(ts); + udf_disk_stamp_to_time(&inode->i_atime, fe->accessTime); + udf_disk_stamp_to_time(&inode->i_mtime, fe->modificationTime); + udf_disk_stamp_to_time(&inode->i_ctime, fe->attrTime); iinfo->i_unique = le64_to_cpu(fe->uniqueID); iinfo->i_lenEAttr = le32_to_cpu(fe->lengthExtendedAttr); @@ -1458,13 +1454,10 @@ reread: inode->i_blocks = le64_to_cpu(efe->logicalBlocksRecorded) << (inode->i_sb->s_blocksize_bits - 9); - udf_disk_stamp_to_time(&ts, efe->accessTime); - inode->i_atime = timespec_to_timespec64(ts); - udf_disk_stamp_to_time(&ts, efe->modificationTime); - inode->i_mtime = timespec_to_timespec64(ts); + udf_disk_stamp_to_time(&inode->i_atime, efe->accessTime); + udf_disk_stamp_to_time(&inode->i_mtime, efe->modificationTime); udf_disk_stamp_to_time(&iinfo->i_crtime, efe->createTime); - udf_disk_stamp_to_time(&ts, efe->attrTime); - inode->i_ctime = timespec_to_timespec64(ts); + udf_disk_stamp_to_time(&inode->i_ctime, efe->attrTime); iinfo->i_unique = le64_to_cpu(efe->uniqueID); iinfo->i_lenEAttr = le32_to_cpu(efe->lengthExtendedAttr); @@ -1601,7 +1594,7 @@ static int udf_sync_inode(struct inode *inode) return udf_update_inode(inode, 1); } -static void udf_adjust_time(struct udf_inode_info *iinfo, struct timespec time) +static void udf_adjust_time(struct udf_inode_info *iinfo, struct timespec64 time) { if (iinfo->i_crtime.tv_sec > time.tv_sec || (iinfo->i_crtime.tv_sec == time.tv_sec && @@ -1714,12 +1707,9 @@ static int udf_update_inode(struct inode *inode, int do_sync) inode->i_sb->s_blocksize - sizeof(struct fileEntry)); fe->logicalBlocksRecorded = cpu_to_le64(lb_recorded); - udf_time_to_disk_stamp(&fe->accessTime, - timespec64_to_timespec(inode->i_atime)); - udf_time_to_disk_stamp(&fe->modificationTime, - timespec64_to_timespec(inode->i_mtime)); - udf_time_to_disk_stamp(&fe->attrTime, - timespec64_to_timespec(inode->i_ctime)); + udf_time_to_disk_stamp(&fe->accessTime, inode->i_atime); + udf_time_to_disk_stamp(&fe->modificationTime, inode->i_mtime); + udf_time_to_disk_stamp(&fe->attrTime, inode->i_ctime); memset(&(fe->impIdent), 0, sizeof(struct regid)); strcpy(fe->impIdent.ident, UDF_ID_DEVELOPER); fe->impIdent.identSuffix[0] = UDF_OS_CLASS_UNIX; @@ -1738,17 +1728,14 @@ static int udf_update_inode(struct inode *inode, int do_sync) efe->objectSize = cpu_to_le64(inode->i_size); efe->logicalBlocksRecorded = cpu_to_le64(lb_recorded); - udf_adjust_time(iinfo, timespec64_to_timespec(inode->i_atime)); - udf_adjust_time(iinfo, timespec64_to_timespec(inode->i_mtime)); - udf_adjust_time(iinfo, timespec64_to_timespec(inode->i_ctime)); + udf_adjust_time(iinfo, inode->i_atime); + udf_adjust_time(iinfo, inode->i_mtime); + udf_adjust_time(iinfo, inode->i_ctime); - udf_time_to_disk_stamp(&efe->accessTime, - timespec64_to_timespec(inode->i_atime)); - udf_time_to_disk_stamp(&efe->modificationTime, - timespec64_to_timespec(inode->i_mtime)); + udf_time_to_disk_stamp(&efe->accessTime, inode->i_atime); + udf_time_to_disk_stamp(&efe->modificationTime, inode->i_mtime); udf_time_to_disk_stamp(&efe->createTime, iinfo->i_crtime); - udf_time_to_disk_stamp(&efe->attrTime, - timespec64_to_timespec(inode->i_ctime)); + udf_time_to_disk_stamp(&efe->attrTime, inode->i_ctime); memset(&(efe->impIdent), 0, sizeof(efe->impIdent)); strcpy(efe->impIdent.ident, UDF_ID_DEVELOPER); diff --git a/fs/udf/super.c b/fs/udf/super.c index 0c504c8031d3..3040dc2a32f6 100644 --- a/fs/udf/super.c +++ b/fs/udf/super.c @@ -1980,7 +1980,7 @@ static void udf_open_lvid(struct super_block *sb) struct buffer_head *bh = sbi->s_lvid_bh; struct logicalVolIntegrityDesc *lvid; struct logicalVolIntegrityDescImpUse *lvidiu; - struct timespec ts; + struct timespec64 ts; if (!bh) return; @@ -1992,7 +1992,7 @@ static void udf_open_lvid(struct super_block *sb) mutex_lock(&sbi->s_alloc_mutex); lvidiu->impIdent.identSuffix[0] = UDF_OS_CLASS_UNIX; lvidiu->impIdent.identSuffix[1] = UDF_OS_ID_LINUX; - ktime_get_real_ts(&ts); + ktime_get_real_ts64(&ts); udf_time_to_disk_stamp(&lvid->recordingDateAndTime, ts); if (le32_to_cpu(lvid->integrityType) == LVID_INTEGRITY_TYPE_CLOSE) lvid->integrityType = cpu_to_le32(LVID_INTEGRITY_TYPE_OPEN); @@ -2017,7 +2017,7 @@ static void udf_close_lvid(struct super_block *sb) struct buffer_head *bh = sbi->s_lvid_bh; struct logicalVolIntegrityDesc *lvid; struct logicalVolIntegrityDescImpUse *lvidiu; - struct timespec ts; + struct timespec64 ts; if (!bh) return; @@ -2029,7 +2029,7 @@ static void udf_close_lvid(struct super_block *sb) mutex_lock(&sbi->s_alloc_mutex); lvidiu->impIdent.identSuffix[0] = UDF_OS_CLASS_UNIX; lvidiu->impIdent.identSuffix[1] = UDF_OS_ID_LINUX; - ktime_get_real_ts(&ts); + ktime_get_real_ts64(&ts); udf_time_to_disk_stamp(&lvid->recordingDateAndTime, ts); if (UDF_MAX_WRITE_VERSION > le16_to_cpu(lvidiu->maxUDFWriteRev)) lvidiu->maxUDFWriteRev = cpu_to_le16(UDF_MAX_WRITE_VERSION); diff --git a/fs/udf/udf_i.h b/fs/udf/udf_i.h index 630426ffb775..2ef0e212f08a 100644 --- a/fs/udf/udf_i.h +++ b/fs/udf/udf_i.h @@ -28,7 +28,7 @@ struct udf_ext_cache { */ struct udf_inode_info { - struct timespec i_crtime; + struct timespec64 i_crtime; /* Physical address of inode */ struct kernel_lb_addr i_location; __u64 i_unique; diff --git a/fs/udf/udf_sb.h b/fs/udf/udf_sb.h index 9dd3e1b9619e..9424d7cab790 100644 --- a/fs/udf/udf_sb.h +++ b/fs/udf/udf_sb.h @@ -138,7 +138,7 @@ struct udf_sb_info { rwlock_t s_cred_lock; /* Root Info */ - struct timespec s_record_time; + struct timespec64 s_record_time; /* Fileset Info */ __u16 s_serial_number; diff --git a/fs/udf/udfdecl.h b/fs/udf/udfdecl.h index 84c47dde4d26..ee246769dee4 100644 --- a/fs/udf/udfdecl.h +++ b/fs/udf/udfdecl.h @@ -258,8 +258,8 @@ extern struct long_ad *udf_get_filelongad(uint8_t *, int, uint32_t *, int); extern struct short_ad *udf_get_fileshortad(uint8_t *, int, uint32_t *, int); /* udftime.c */ -extern void udf_disk_stamp_to_time(struct timespec *dest, +extern void udf_disk_stamp_to_time(struct timespec64 *dest, struct timestamp src); -extern void udf_time_to_disk_stamp(struct timestamp *dest, struct timespec src); +extern void udf_time_to_disk_stamp(struct timestamp *dest, struct timespec64 src); #endif /* __UDF_DECL_H */ diff --git a/fs/udf/udftime.c b/fs/udf/udftime.c index 67b33ac5d41b..fce4ad976c8c 100644 --- a/fs/udf/udftime.c +++ b/fs/udf/udftime.c @@ -41,7 +41,7 @@ #include <linux/time.h> void -udf_disk_stamp_to_time(struct timespec *dest, struct timestamp src) +udf_disk_stamp_to_time(struct timespec64 *dest, struct timestamp src) { u16 typeAndTimezone = le16_to_cpu(src.typeAndTimezone); u16 year = le16_to_cpu(src.year); @@ -70,9 +70,9 @@ udf_disk_stamp_to_time(struct timespec *dest, struct timestamp src) } void -udf_time_to_disk_stamp(struct timestamp *dest, struct timespec ts) +udf_time_to_disk_stamp(struct timestamp *dest, struct timespec64 ts) { - long seconds; + time64_t seconds; int16_t offset; struct tm tm; diff --git a/include/linux/dm-kcopyd.h b/include/linux/dm-kcopyd.h index cfac8588ed56..e42de7750c88 100644 --- a/include/linux/dm-kcopyd.h +++ b/include/linux/dm-kcopyd.h @@ -62,9 +62,9 @@ void dm_kcopyd_client_destroy(struct dm_kcopyd_client *kc); typedef void (*dm_kcopyd_notify_fn)(int read_err, unsigned long write_err, void *context); -int dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, - unsigned num_dests, struct dm_io_region *dests, - unsigned flags, dm_kcopyd_notify_fn fn, void *context); +void dm_kcopyd_copy(struct dm_kcopyd_client *kc, struct dm_io_region *from, + unsigned num_dests, struct dm_io_region *dests, + unsigned flags, dm_kcopyd_notify_fn fn, void *context); /* * Prepare a callback and submit it via the kcopyd thread. @@ -81,9 +81,9 @@ void *dm_kcopyd_prepare_callback(struct dm_kcopyd_client *kc, dm_kcopyd_notify_fn fn, void *context); void dm_kcopyd_do_callback(void *job, int read_err, unsigned long write_err); -int dm_kcopyd_zero(struct dm_kcopyd_client *kc, - unsigned num_dests, struct dm_io_region *dests, - unsigned flags, dm_kcopyd_notify_fn fn, void *context); +void dm_kcopyd_zero(struct dm_kcopyd_client *kc, + unsigned num_dests, struct dm_io_region *dests, + unsigned flags, dm_kcopyd_notify_fn fn, void *context); #endif /* __KERNEL__ */ #endif /* _LINUX_DM_KCOPYD_H */ diff --git a/include/linux/fsnotify_backend.h b/include/linux/fsnotify_backend.h index b38964a7a521..2b9b6f1ff5e0 100644 --- a/include/linux/fsnotify_backend.h +++ b/include/linux/fsnotify_backend.h @@ -210,6 +210,11 @@ enum fsnotify_obj_type { #define FSNOTIFY_OBJ_TYPE_VFSMOUNT_FL (1U << FSNOTIFY_OBJ_TYPE_VFSMOUNT) #define FSNOTIFY_OBJ_ALL_TYPES_MASK ((1U << FSNOTIFY_OBJ_TYPE_COUNT) - 1) +static inline bool fsnotify_valid_obj_type(unsigned int type) +{ + return (type < FSNOTIFY_OBJ_TYPE_COUNT); +} + struct fsnotify_iter_info { struct fsnotify_mark *marks[FSNOTIFY_OBJ_TYPE_COUNT]; unsigned int report_mask; @@ -251,6 +256,13 @@ FSNOTIFY_ITER_FUNCS(vfsmount, VFSMOUNT) for (type = 0; type < FSNOTIFY_OBJ_TYPE_COUNT; type++) /* + * fsnotify_connp_t is what we embed in objects which connector can be attached + * to. fsnotify_connp_t * is how we refer from connector back to object. + */ +struct fsnotify_mark_connector; +typedef struct fsnotify_mark_connector __rcu *fsnotify_connp_t; + +/* * Inode / vfsmount point to this structure which tracks all marks attached to * the inode / vfsmount. The reference to inode / vfsmount is held by this * structure. We destroy this structure when there are no more marks attached @@ -259,9 +271,9 @@ FSNOTIFY_ITER_FUNCS(vfsmount, VFSMOUNT) struct fsnotify_mark_connector { spinlock_t lock; unsigned int type; /* Type of object [lock] */ - union { /* Object pointer [lock] */ - struct inode *inode; - struct vfsmount *mnt; + union { + /* Object pointer [lock] */ + fsnotify_connp_t *obj; /* Used listing heads to free after srcu period expires */ struct fsnotify_mark_connector *destroy_next; }; @@ -389,32 +401,36 @@ extern struct fsnotify_event *fsnotify_remove_first_event(struct fsnotify_group /* functions used to manipulate the marks attached to inodes */ +/* Get mask of events for a list of marks */ +extern __u32 fsnotify_conn_mask(struct fsnotify_mark_connector *conn); /* Calculate mask of events for a list of marks */ extern void fsnotify_recalc_mask(struct fsnotify_mark_connector *conn); extern void fsnotify_init_mark(struct fsnotify_mark *mark, struct fsnotify_group *group); /* Find mark belonging to given group in the list of marks */ -extern struct fsnotify_mark *fsnotify_find_mark( - struct fsnotify_mark_connector __rcu **connp, - struct fsnotify_group *group); -/* attach the mark to the inode or vfsmount */ -extern int fsnotify_add_mark(struct fsnotify_mark *mark, struct inode *inode, - struct vfsmount *mnt, int allow_dups); +extern struct fsnotify_mark *fsnotify_find_mark(fsnotify_connp_t *connp, + struct fsnotify_group *group); +/* attach the mark to the object */ +extern int fsnotify_add_mark(struct fsnotify_mark *mark, + fsnotify_connp_t *connp, unsigned int type, + int allow_dups); extern int fsnotify_add_mark_locked(struct fsnotify_mark *mark, - struct inode *inode, struct vfsmount *mnt, + fsnotify_connp_t *connp, unsigned int type, int allow_dups); /* attach the mark to the inode */ static inline int fsnotify_add_inode_mark(struct fsnotify_mark *mark, struct inode *inode, int allow_dups) { - return fsnotify_add_mark(mark, inode, NULL, allow_dups); + return fsnotify_add_mark(mark, &inode->i_fsnotify_marks, + FSNOTIFY_OBJ_TYPE_INODE, allow_dups); } static inline int fsnotify_add_inode_mark_locked(struct fsnotify_mark *mark, struct inode *inode, int allow_dups) { - return fsnotify_add_mark_locked(mark, inode, NULL, allow_dups); + return fsnotify_add_mark_locked(mark, &inode->i_fsnotify_marks, + FSNOTIFY_OBJ_TYPE_INODE, allow_dups); } /* given a group and a mark, flag mark to be freed when all references are dropped */ extern void fsnotify_destroy_mark(struct fsnotify_mark *mark, diff --git a/include/linux/inotify.h b/include/linux/inotify.h index 44f9ffe72c87..6a24905f6e1e 100644 --- a/include/linux/inotify.h +++ b/include/linux/inotify.h @@ -18,6 +18,6 @@ extern struct ctl_table inotify_table[]; /* for sysctl */ IN_DELETE_SELF | IN_MOVE_SELF | IN_UNMOUNT | \ IN_Q_OVERFLOW | IN_IGNORED | IN_ONLYDIR | \ IN_DONT_FOLLOW | IN_EXCL_UNLINK | IN_MASK_ADD | \ - IN_ISDIR | IN_ONESHOT) + IN_MASK_CREATE | IN_ISDIR | IN_ONESHOT) #endif /* _LINUX_INOTIFY_H */ diff --git a/include/linux/mm_types.h b/include/linux/mm_types.h index efdc24dd9e97..cd2bc939efd0 100644 --- a/include/linux/mm_types.h +++ b/include/linux/mm_types.h @@ -139,7 +139,10 @@ struct page { unsigned long _pt_pad_1; /* compound_head */ pgtable_t pmd_huge_pte; /* protected by page->ptl */ unsigned long _pt_pad_2; /* mapping */ - struct mm_struct *pt_mm; /* x86 pgds only */ + union { + struct mm_struct *pt_mm; /* x86 pgds only */ + atomic_t pt_frag_refcount; /* powerpc */ + }; #if ALLOC_SPLIT_PTLOCKS spinlock_t *ptl; #else diff --git a/include/linux/module.h b/include/linux/module.h index d44df9b2c131..f807f15bebbe 100644 --- a/include/linux/module.h +++ b/include/linux/module.h @@ -266,7 +266,7 @@ extern int modules_disabled; /* for sysctl */ /* Get/put a kernel symbol (calls must be symmetric) */ void *__symbol_get(const char *symbol); void *__symbol_get_gpl(const char *symbol); -#define symbol_get(x) ((typeof(&x))(__symbol_get(VMLINUX_SYMBOL_STR(x)))) +#define symbol_get(x) ((typeof(&x))(__symbol_get(__stringify(x)))) /* modules using other modules: kdb wants to see this. */ struct module_use { @@ -575,7 +575,7 @@ extern void __noreturn __module_put_and_exit(struct module *mod, #ifdef CONFIG_MODULE_UNLOAD int module_refcount(struct module *mod); void __symbol_put(const char *symbol); -#define symbol_put(x) __symbol_put(VMLINUX_SYMBOL_STR(x)) +#define symbol_put(x) __symbol_put(__stringify(x)) void symbol_put_addr(void *addr); /* Sometimes we know we already have a refcount, and it's easier not diff --git a/include/misc/cxl-base.h b/include/misc/cxl-base.h index b2ebc91fe09a..f53808fa638a 100644 --- a/include/misc/cxl-base.h +++ b/include/misc/cxl-base.h @@ -10,8 +10,6 @@ #ifndef _MISC_CXL_BASE_H #define _MISC_CXL_BASE_H -#include <misc/cxl.h> - #ifdef CONFIG_CXL_BASE #define CXL_IRQ_RANGES 4 @@ -41,10 +39,6 @@ static inline void cxl_ctx_put(void) struct cxl_afu *cxl_afu_get(struct cxl_afu *afu); void cxl_afu_put(struct cxl_afu *afu); void cxl_slbia(struct mm_struct *mm); -bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu); -void cxl_pci_disable_device(struct pci_dev *dev); -int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type); -void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev); #else /* CONFIG_CXL_BASE */ @@ -52,10 +46,6 @@ static inline bool cxl_ctx_in_use(void) { return false; } static inline struct cxl_afu *cxl_afu_get(struct cxl_afu *afu) { return NULL; } static inline void cxl_afu_put(struct cxl_afu *afu) {} static inline void cxl_slbia(struct mm_struct *mm) {} -static inline bool cxl_pci_associate_default_context(struct pci_dev *dev, struct cxl_afu *afu) { return false; } -static inline void cxl_pci_disable_device(struct pci_dev *dev) {} -static inline int cxl_cx4_setup_msi_irqs(struct pci_dev *pdev, int nvec, int type) { return -ENODEV; } -static inline void cxl_cx4_teardown_msi_irqs(struct pci_dev *pdev) {} #endif /* CONFIG_CXL_BASE */ diff --git a/include/misc/cxl.h b/include/misc/cxl.h index b712be544f8c..ea9ff4a1a9ca 100644 --- a/include/misc/cxl.h +++ b/include/misc/cxl.h @@ -24,46 +24,6 @@ * generic PCI API. This API is agnostic to the actual AFU. */ -#define CXL_SLOT_FLAG_DMA 0x1 - -/* - * Checks if the given card is in a cxl capable slot. Pass CXL_SLOT_FLAG_DMA if - * the card requires CAPP DMA mode to also check if the system supports it. - * This is intended to be used by bi-modal devices to determine if they can use - * cxl mode or if they should continue running in PCI mode. - * - * Note that this only checks if the slot is cxl capable - it does not - * currently check if the CAPP is currently available for chips where it can be - * assigned to different PHBs on a first come first serve basis (i.e. P8) - */ -bool cxl_slot_is_supported(struct pci_dev *dev, int flags); - - -#define CXL_BIMODE_CXL 1 -#define CXL_BIMODE_PCI 2 - -/* - * Check the mode that the given bi-modal CXL adapter is currently in and - * change it if necessary. This does not apply to AFU drivers. - * - * If the mode matches the requested mode this function will return 0 - if the - * driver was expecting the generic CXL driver to have bound to the adapter and - * it gets this return value it should fail the probe function to give the CXL - * driver a chance to probe it. - * - * If the mode does not match it will start a background task to unplug the - * device from Linux and switch its mode, and will return -EBUSY. At this - * point the calling driver should make sure it has released the device and - * fail its probe function. - * - * The offset of the CXL VSEC can be provided to this function. If 0 is passed, - * this function will search for a CXL VSEC with ID 0x1280 and return -ENODEV - * if it is not found. - */ -#ifdef CONFIG_CXL_BIMODAL -int cxl_check_and_switch_mode(struct pci_dev *dev, int mode, int vsec); -#endif - /* Get the AFU associated with a pci_dev */ struct cxl_afu *cxl_pci_to_afu(struct pci_dev *dev); @@ -174,14 +134,6 @@ int cxl_afu_reset(struct cxl_context *ctx); void cxl_set_master(struct cxl_context *ctx); /* - * Sets the context to use real mode memory accesses to operate with - * translation disabled. Note that this only makes sense for kernel contexts - * under bare metal, and will not work with virtualisation. May only be - * performed on stopped contexts. - */ -int cxl_set_translation_mode(struct cxl_context *ctx, bool real_mode); - -/* * Map and unmap the AFU Problem Space area. The amount and location mapped * depends on if this context is a master or slave. */ @@ -192,26 +144,6 @@ void cxl_psa_unmap(void __iomem *addr); int cxl_process_element(struct cxl_context *ctx); /* - * Limit the number of interrupts that a single context can allocate via - * cxl_start_work. If using the api with a real phb, this may be used to - * request that additional default contexts be created when allocating - * interrupts via pci_enable_msix_range. These will be set to the same running - * state as the default context, and if that is running it will reuse the - * parameters previously passed to cxl_start_context for the default context. - */ -int cxl_set_max_irqs_per_process(struct pci_dev *dev, int irqs); -int cxl_get_max_irqs_per_process(struct pci_dev *dev); - -/* - * Use to simultaneously iterate over hardware interrupt numbers, contexts and - * afu interrupt numbers allocated for the device via pci_enable_msix_range and - * is a useful convenience function when working with hardware that has - * limitations on the number of interrupts per process. *ctx and *afu_irq - * should be NULL and 0 to start the iteration. - */ -int cxl_next_msi_hwirq(struct pci_dev *pdev, struct cxl_context **ctx, int *afu_irq); - -/* * These calls allow drivers to create their own file descriptors and make them * identical to the cxl file descriptor user API. An example use case: * diff --git a/include/uapi/linux/inotify.h b/include/uapi/linux/inotify.h index 4800bf2a531d..884b4846b630 100644 --- a/include/uapi/linux/inotify.h +++ b/include/uapi/linux/inotify.h @@ -53,6 +53,7 @@ struct inotify_event { #define IN_ONLYDIR 0x01000000 /* only watch the path if it is a directory */ #define IN_DONT_FOLLOW 0x02000000 /* don't follow a sym link */ #define IN_EXCL_UNLINK 0x04000000 /* exclude events on unlinked objects */ +#define IN_MASK_CREATE 0x10000000 /* only create watches */ #define IN_MASK_ADD 0x20000000 /* add to the mask of an already existing watch */ #define IN_ISDIR 0x40000000 /* event occurred against dir */ #define IN_ONESHOT 0x80000000 /* only send event once */ diff --git a/include/uapi/linux/pmu.h b/include/uapi/linux/pmu.h index 89cb1acea93a..97256f90e6df 100644 --- a/include/uapi/linux/pmu.h +++ b/include/uapi/linux/pmu.h @@ -93,8 +93,8 @@ enum { PMU_HEATHROW_BASED, /* PowerBook G3 series */ PMU_PADDINGTON_BASED, /* 1999 PowerBook G3 */ PMU_KEYLARGO_BASED, /* Core99 motherboard (PMU99) */ - PMU_68K_V1, /* 68K PMU, version 1 */ - PMU_68K_V2, /* 68K PMU, version 2 */ + PMU_68K_V1, /* Unused/deprecated */ + PMU_68K_V2, /* Unused/deprecated */ }; /* PMU PMU_POWER_EVENTS commands */ diff --git a/kernel/audit_tree.c b/kernel/audit_tree.c index 9f6eaeb6919f..ea43181cde4a 100644 --- a/kernel/audit_tree.c +++ b/kernel/audit_tree.c @@ -168,7 +168,8 @@ static __cacheline_aligned_in_smp DEFINE_SPINLOCK(hash_lock); /* Function to return search key in our hash from inode. */ static unsigned long inode_to_key(const struct inode *inode) { - return (unsigned long)inode; + /* Use address pointed to by connector->obj as the key */ + return (unsigned long)&inode->i_fsnotify_marks; } /* @@ -183,7 +184,7 @@ static unsigned long chunk_to_key(struct audit_chunk *chunk) */ if (WARN_ON_ONCE(!chunk->mark.connector)) return 0; - return (unsigned long)chunk->mark.connector->inode; + return (unsigned long)chunk->mark.connector->obj; } static inline struct list_head *chunk_hash(unsigned long key) @@ -258,7 +259,7 @@ static void untag_chunk(struct node *p) spin_lock(&entry->lock); /* * mark_mutex protects mark from getting detached and thus also from - * mark->connector->inode getting NULL. + * mark->connector->obj getting NULL. */ if (chunk->dead || !(entry->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) { spin_unlock(&entry->lock); @@ -288,8 +289,8 @@ static void untag_chunk(struct node *p) if (!new) goto Fallback; - if (fsnotify_add_inode_mark_locked(&new->mark, entry->connector->inode, - 1)) { + if (fsnotify_add_mark_locked(&new->mark, entry->connector->obj, + FSNOTIFY_OBJ_TYPE_INODE, 1)) { fsnotify_put_mark(&new->mark); goto Fallback; } @@ -423,7 +424,7 @@ static int tag_chunk(struct inode *inode, struct audit_tree *tree) spin_lock(&old_entry->lock); /* * mark_mutex protects mark from getting detached and thus also from - * mark->connector->inode getting NULL. + * mark->connector->obj getting NULL. */ if (!(old_entry->flags & FSNOTIFY_MARK_FLAG_ATTACHED)) { /* old_entry is being shot, lets just lie */ @@ -434,8 +435,8 @@ static int tag_chunk(struct inode *inode, struct audit_tree *tree) return -ENOENT; } - if (fsnotify_add_inode_mark_locked(chunk_entry, - old_entry->connector->inode, 1)) { + if (fsnotify_add_mark_locked(chunk_entry, old_entry->connector->obj, + FSNOTIFY_OBJ_TYPE_INODE, 1)) { spin_unlock(&old_entry->lock); mutex_unlock(&old_entry->group->mark_mutex); fsnotify_put_mark(chunk_entry); diff --git a/kernel/module-internal.h b/kernel/module-internal.h index 915e123a430f..79c9be2dbbe9 100644 --- a/kernel/module-internal.h +++ b/kernel/module-internal.h @@ -9,4 +9,27 @@ * 2 of the Licence, or (at your option) any later version. */ -extern int mod_verify_sig(const void *mod, unsigned long *_modlen); +#include <linux/elf.h> +#include <asm/module.h> + +struct load_info { + const char *name; + /* pointer to module in temporary copy, freed at end of load_module() */ + struct module *mod; + Elf_Ehdr *hdr; + unsigned long len; + Elf_Shdr *sechdrs; + char *secstrings, *strtab; + unsigned long symoffs, stroffs; + struct _ddebug *debug; + unsigned int num_debug; + bool sig_ok; +#ifdef CONFIG_KALLSYMS + unsigned long mod_kallsyms_init_off; +#endif + struct { + unsigned int sym, str, mod, vers, info, pcpu; + } index; +}; + +extern int mod_verify_sig(const void *mod, struct load_info *info); diff --git a/kernel/module.c b/kernel/module.c index a7615d661910..b046a32520d8 100644 --- a/kernel/module.c +++ b/kernel/module.c @@ -307,24 +307,6 @@ int unregister_module_notifier(struct notifier_block *nb) } EXPORT_SYMBOL(unregister_module_notifier); -struct load_info { - const char *name; - Elf_Ehdr *hdr; - unsigned long len; - Elf_Shdr *sechdrs; - char *secstrings, *strtab; - unsigned long symoffs, stroffs; - struct _ddebug *debug; - unsigned int num_debug; - bool sig_ok; -#ifdef CONFIG_KALLSYMS - unsigned long mod_kallsyms_init_off; -#endif - struct { - unsigned int sym, str, mod, vers, info, pcpu; - } index; -}; - /* * We require a truly strong try_module_get(): 0 means success. * Otherwise an error is returned due to ongoing or failed @@ -1339,14 +1321,12 @@ static inline int check_modstruct_version(const struct load_info *info, * locking is necessary -- use preempt_disable() to placate lockdep. */ preempt_disable(); - if (!find_symbol(VMLINUX_SYMBOL_STR(module_layout), NULL, - &crc, true, false)) { + if (!find_symbol("module_layout", NULL, &crc, true, false)) { preempt_enable(); BUG(); } preempt_enable(); - return check_version(info, VMLINUX_SYMBOL_STR(module_layout), - mod, crc); + return check_version(info, "module_layout", mod, crc); } /* First part is kernel version, which we ignore if module has crcs. */ @@ -2059,21 +2039,19 @@ static int copy_module_elf(struct module *mod, struct load_info *info) /* Elf section header table */ size = sizeof(*info->sechdrs) * info->hdr->e_shnum; - mod->klp_info->sechdrs = kmalloc(size, GFP_KERNEL); + mod->klp_info->sechdrs = kmemdup(info->sechdrs, size, GFP_KERNEL); if (mod->klp_info->sechdrs == NULL) { ret = -ENOMEM; goto free_info; } - memcpy(mod->klp_info->sechdrs, info->sechdrs, size); /* Elf section name string table */ size = info->sechdrs[info->hdr->e_shstrndx].sh_size; - mod->klp_info->secstrings = kmalloc(size, GFP_KERNEL); + mod->klp_info->secstrings = kmemdup(info->secstrings, size, GFP_KERNEL); if (mod->klp_info->secstrings == NULL) { ret = -ENOMEM; goto free_sechdrs; } - memcpy(mod->klp_info->secstrings, info->secstrings, size); /* Elf symbol section index */ symndx = info->index.sym; @@ -2282,9 +2260,9 @@ static int simplify_symbols(struct module *mod, const struct load_info *info) if (!ksym && ELF_ST_BIND(sym[i].st_info) == STB_WEAK) break; - pr_warn("%s: Unknown symbol %s (err %li)\n", - mod->name, name, PTR_ERR(ksym)); ret = PTR_ERR(ksym) ?: -ENOENT; + pr_warn("%s: Unknown symbol %s (err %d)\n", + mod->name, name, ret); break; default: @@ -2486,7 +2464,11 @@ static char *get_modinfo(struct load_info *info, const char *tag) Elf_Shdr *infosec = &info->sechdrs[info->index.info]; unsigned long size = infosec->sh_size; - for (p = (char *)infosec->sh_addr; p; p = next_string(p, &size)) { + /* + * get_modinfo() calls made before rewrite_section_headers() + * must use sh_offset, as sh_addr isn't set! + */ + for (p = (char *)info->hdr + infosec->sh_offset; p; p = next_string(p, &size)) { if (strncmp(p, tag, taglen) == 0 && p[taglen] == '=') return p + taglen + 1; } @@ -2774,7 +2756,7 @@ static int module_sig_check(struct load_info *info, int flags) memcmp(mod + info->len - markerlen, MODULE_SIG_STRING, markerlen) == 0) { /* We truncate the module to discard the signature */ info->len -= markerlen; - err = mod_verify_sig(mod, &info->len); + err = mod_verify_sig(mod, info); } if (!err) { @@ -2926,17 +2908,7 @@ static int rewrite_section_headers(struct load_info *info, int flags) } /* Track but don't keep modinfo and version sections. */ - if (flags & MODULE_INIT_IGNORE_MODVERSIONS) - info->index.vers = 0; /* Pretend no __versions section! */ - else - info->index.vers = find_sec(info, "__versions"); info->sechdrs[info->index.vers].sh_flags &= ~(unsigned long)SHF_ALLOC; - - info->index.info = find_sec(info, ".modinfo"); - if (!info->index.info) - info->name = "(missing .modinfo section)"; - else - info->name = get_modinfo(info, "name"); info->sechdrs[info->index.info].sh_flags &= ~(unsigned long)SHF_ALLOC; return 0; @@ -2947,23 +2919,24 @@ static int rewrite_section_headers(struct load_info *info, int flags) * search for module section index etc), and do some basic section * verification. * - * Return the temporary module pointer (we'll replace it with the final - * one when we move the module sections around). + * Set info->mod to the temporary copy of the module in info->hdr. The final one + * will be allocated in move_module(). */ -static struct module *setup_load_info(struct load_info *info, int flags) +static int setup_load_info(struct load_info *info, int flags) { unsigned int i; - int err; - struct module *mod; /* Set up the convenience variables */ info->sechdrs = (void *)info->hdr + info->hdr->e_shoff; info->secstrings = (void *)info->hdr + info->sechdrs[info->hdr->e_shstrndx].sh_offset; - err = rewrite_section_headers(info, flags); - if (err) - return ERR_PTR(err); + /* Try to find a name early so we can log errors with a module name */ + info->index.info = find_sec(info, ".modinfo"); + if (!info->index.info) + info->name = "(missing .modinfo section)"; + else + info->name = get_modinfo(info, "name"); /* Find internal symbols and strings. */ for (i = 1; i < info->hdr->e_shnum; i++) { @@ -2976,34 +2949,35 @@ static struct module *setup_load_info(struct load_info *info, int flags) } } + if (info->index.sym == 0) { + pr_warn("%s: module has no symbols (stripped?)\n", info->name); + return -ENOEXEC; + } + info->index.mod = find_sec(info, ".gnu.linkonce.this_module"); if (!info->index.mod) { pr_warn("%s: No module found in object\n", info->name ?: "(missing .modinfo name field)"); - return ERR_PTR(-ENOEXEC); + return -ENOEXEC; } /* This is temporary: point mod into copy of data. */ - mod = (void *)info->sechdrs[info->index.mod].sh_addr; + info->mod = (void *)info->hdr + info->sechdrs[info->index.mod].sh_offset; /* - * If we didn't load the .modinfo 'name' field, fall back to + * If we didn't load the .modinfo 'name' field earlier, fall back to * on-disk struct mod 'name' field. */ if (!info->name) - info->name = mod->name; + info->name = info->mod->name; - if (info->index.sym == 0) { - pr_warn("%s: module has no symbols (stripped?)\n", info->name); - return ERR_PTR(-ENOEXEC); - } + if (flags & MODULE_INIT_IGNORE_MODVERSIONS) + info->index.vers = 0; /* Pretend no __versions section! */ + else + info->index.vers = find_sec(info, "__versions"); info->index.pcpu = find_pcpusec(info); - /* Check module struct version now, before we try to use module. */ - if (!check_modstruct_version(info, mod)) - return ERR_PTR(-ENOEXEC); - - return mod; + return 0; } static int check_modinfo(struct module *mod, struct load_info *info, int flags) @@ -3298,25 +3272,17 @@ core_param(module_blacklist, module_blacklist, charp, 0400); static struct module *layout_and_allocate(struct load_info *info, int flags) { - /* Module within temporary copy. */ struct module *mod; unsigned int ndx; int err; - mod = setup_load_info(info, flags); - if (IS_ERR(mod)) - return mod; - - if (blacklisted(info->name)) - return ERR_PTR(-EPERM); - - err = check_modinfo(mod, info, flags); + err = check_modinfo(info->mod, info, flags); if (err) return ERR_PTR(err); /* Allow arches to frob section contents and sizes. */ err = module_frob_arch_sections(info->hdr, info->sechdrs, - info->secstrings, mod); + info->secstrings, info->mod); if (err < 0) return ERR_PTR(err); @@ -3335,11 +3301,11 @@ static struct module *layout_and_allocate(struct load_info *info, int flags) /* Determine total sizes, and put offsets in sh_entsize. For now this is done generically; there doesn't appear to be any special cases for the architectures. */ - layout_sections(mod, info); - layout_symtab(mod, info); + layout_sections(info->mod, info); + layout_symtab(info->mod, info); /* Allocate and move to the final place */ - err = move_module(mod, info); + err = move_module(info->mod, info); if (err) return ERR_PTR(err); @@ -3657,17 +3623,36 @@ static int load_module(struct load_info *info, const char __user *uargs, int flags) { struct module *mod; - long err; + long err = 0; char *after_dashes; + err = elf_header_check(info); + if (err) + goto free_copy; + + err = setup_load_info(info, flags); + if (err) + goto free_copy; + + if (blacklisted(info->name)) { + err = -EPERM; + goto free_copy; + } + err = module_sig_check(info, flags); if (err) goto free_copy; - err = elf_header_check(info); + err = rewrite_section_headers(info, flags); if (err) goto free_copy; + /* Check module struct version now, before we try to use module. */ + if (!check_modstruct_version(info, info->mod)) { + err = -ENOEXEC; + goto free_copy; + } + /* Figure out module layout, and allocate all the memory. */ mod = layout_and_allocate(info, flags); if (IS_ERR(mod)) { @@ -4067,7 +4052,7 @@ static unsigned long mod_find_symname(struct module *mod, const char *name) for (i = 0; i < kallsyms->num_symtab; i++) if (strcmp(name, symname(kallsyms, i)) == 0 && - kallsyms->symtab[i].st_info != 'U') + kallsyms->symtab[i].st_shndx != SHN_UNDEF) return kallsyms->symtab[i].st_value; return 0; } @@ -4113,6 +4098,10 @@ int module_kallsyms_on_each_symbol(int (*fn)(void *, const char *, if (mod->state == MODULE_STATE_UNFORMED) continue; for (i = 0; i < kallsyms->num_symtab; i++) { + + if (kallsyms->symtab[i].st_shndx == SHN_UNDEF) + continue; + ret = fn(data, symname(kallsyms, i), mod, kallsyms->symtab[i].st_value); if (ret != 0) diff --git a/kernel/module_signing.c b/kernel/module_signing.c index 937c844bee4a..f2075ce8e4b3 100644 --- a/kernel/module_signing.c +++ b/kernel/module_signing.c @@ -45,10 +45,10 @@ struct module_signature { /* * Verify the signature on a module. */ -int mod_verify_sig(const void *mod, unsigned long *_modlen) +int mod_verify_sig(const void *mod, struct load_info *info) { struct module_signature ms; - size_t modlen = *_modlen, sig_len; + size_t sig_len, modlen = info->len; pr_devel("==>%s(,%zu)\n", __func__, modlen); @@ -62,10 +62,11 @@ int mod_verify_sig(const void *mod, unsigned long *_modlen) if (sig_len >= modlen) return -EBADMSG; modlen -= sig_len; - *_modlen = modlen; + info->len = modlen; if (ms.id_type != PKEY_ID_PKCS7) { - pr_err("Module is not signed with expected PKCS#7 message\n"); + pr_err("%s: Module is not signed with expected PKCS#7 message\n", + info->name); return -ENOPKG; } @@ -76,7 +77,8 @@ int mod_verify_sig(const void *mod, unsigned long *_modlen) ms.__pad[0] != 0 || ms.__pad[1] != 0 || ms.__pad[2] != 0) { - pr_err("PKCS#7 signature info has unexpected non-zero params\n"); + pr_err("%s: PKCS#7 signature info has unexpected non-zero params\n", + info->name); return -EBADMSG; } diff --git a/scripts/checkstack.pl b/scripts/checkstack.pl index cbdf0dfd4c22..8081b6cf67d2 100755 --- a/scripts/checkstack.pl +++ b/scripts/checkstack.pl @@ -15,6 +15,7 @@ # M68k port by Geert Uytterhoeven and Andreas Schwab # AArch64, PARISC ports by Kyle McMartin # sparc port by Martin Habets <errandir_news@mph.eclipse.co.uk> +# ppc64le port by Breno Leitao <leitao@debian.org> # # Usage: # objdump -d vmlinux | scripts/checkstack.pl [arch] @@ -78,13 +79,9 @@ my (@stack, $re, $dre, $x, $xs, $funcre); $re = qr/.*l\.addi.*r1,r1,-(([0-9]{2}|[3-9])[0-9]{2})/o; } elsif ($arch eq 'parisc' || $arch eq 'parisc64') { $re = qr/.*ldo ($x{1,8})\(sp\),sp/o; - } elsif ($arch eq 'ppc') { - #c00029f4: 94 21 ff 30 stwu r1,-208(r1) - $re = qr/.*stwu.*r1,-($x{1,8})\(r1\)/o; - } elsif ($arch eq 'ppc64') { - #XXX - $re = qr/.*stdu.*r1,-($x{1,8})\(r1\)/o; - } elsif ($arch eq 'powerpc') { + } elsif ($arch eq 'powerpc' || $arch =~ /^ppc(64)?(le)?$/ ) { + # powerpc : 94 21 ff 30 stwu r1,-208(r1) + # ppc64(le) : 81 ff 21 f8 stdu r1,-128(r1) $re = qr/.*st[dw]u.*r1,-($x{1,8})\(r1\)/o; } elsif ($arch =~ /^s390x?$/) { # 11160: a7 fb ff 60 aghi %r15,-160 diff --git a/sound/ppc/snd_ps3.c b/sound/ppc/snd_ps3.c index 36f34f434ecb..abe031c9d592 100644 --- a/sound/ppc/snd_ps3.c +++ b/sound/ppc/snd_ps3.c @@ -930,6 +930,7 @@ static int snd_ps3_driver_probe(struct ps3_system_bus_device *dev) { int i, ret; u64 lpar_addr, lpar_size; + static u64 dummy_mask; if (WARN_ON(!firmware_has_feature(FW_FEATURE_PS3_LV1))) return -ENODEV; @@ -970,6 +971,10 @@ static int snd_ps3_driver_probe(struct ps3_system_bus_device *dev) goto clean_mmio; } + dummy_mask = DMA_BIT_MASK(32); + dev->core.dma_mask = &dummy_mask; + dma_set_coherent_mask(&dev->core, dummy_mask); + snd_ps3_audio_set_base_addr(dev->d_region->bus_addr); /* CONFIG_SND_PS3_DEFAULT_START_DELAY */ diff --git a/tools/testing/selftests/powerpc/alignment/.gitignore b/tools/testing/selftests/powerpc/alignment/.gitignore index 9d383073b7ad..6d4fd014511c 100644 --- a/tools/testing/selftests/powerpc/alignment/.gitignore +++ b/tools/testing/selftests/powerpc/alignment/.gitignore @@ -1,6 +1,2 @@ -copy_unaligned copy_first_unaligned -paste_unaligned -paste_last_unaligned -copy_paste_unaligned_common alignment_handler diff --git a/tools/testing/selftests/powerpc/alignment/Makefile b/tools/testing/selftests/powerpc/alignment/Makefile index 083a48a008b4..93baacab7693 100644 --- a/tools/testing/selftests/powerpc/alignment/Makefile +++ b/tools/testing/selftests/powerpc/alignment/Makefile @@ -1,6 +1,5 @@ -TEST_GEN_PROGS := copy_unaligned copy_first_unaligned paste_unaligned \ - paste_last_unaligned alignment_handler +TEST_GEN_PROGS := copy_first_unaligned alignment_handler include ../../lib.mk -$(TEST_GEN_PROGS): ../harness.c ../utils.c copy_paste_unaligned_common.c +$(TEST_GEN_PROGS): ../harness.c ../utils.c diff --git a/tools/testing/selftests/powerpc/alignment/alignment_handler.c b/tools/testing/selftests/powerpc/alignment/alignment_handler.c index 0f2698f9fd6d..169a8b9719fb 100644 --- a/tools/testing/selftests/powerpc/alignment/alignment_handler.c +++ b/tools/testing/selftests/powerpc/alignment/alignment_handler.c @@ -40,6 +40,7 @@ #include <sys/stat.h> #include <fcntl.h> #include <unistd.h> +#include <stdbool.h> #include <stdio.h> #include <stdlib.h> #include <string.h> @@ -48,6 +49,8 @@ #include <setjmp.h> #include <signal.h> +#include <asm/cputable.h> + #include "utils.h" int bufsize; @@ -191,7 +194,7 @@ int test_memcmp(void *s1, void *s2, int n, int offset, char *test_name) */ int do_test(char *test_name, void (*test_func)(char *, char *)) { - int offset, width, fd, rc = 0, r; + int offset, width, fd, rc, r; void *mem0, *mem1, *ci0, *ci1; printf("\tDoing %s:\t", test_name); @@ -199,8 +202,8 @@ int do_test(char *test_name, void (*test_func)(char *, char *)) fd = open("/dev/fb0", O_RDWR); if (fd < 0) { printf("\n"); - perror("Can't open /dev/fb0"); - SKIP_IF(1); + perror("Can't open /dev/fb0 now?"); + return 1; } ci0 = mmap(NULL, bufsize, PROT_WRITE, MAP_SHARED, @@ -226,6 +229,7 @@ int do_test(char *test_name, void (*test_func)(char *, char *)) return rc; } + rc = 0; /* offset = 0 no alignment fault, so skip */ for (offset = 1; offset < 16; offset++) { width = 16; /* vsx == 16 bytes */ @@ -244,32 +248,51 @@ int do_test(char *test_name, void (*test_func)(char *, char *)) r |= test_memcpy(mem1, mem0, width, offset, test_func); if (r && !debug) { printf("FAILED: Got signal"); + rc = 1; break; } r |= test_memcmp(mem1, ci1, width, offset, test_name); - rc |= r; if (r && !debug) { printf("FAILED: Wrong Data"); + rc = 1; break; } } - if (!r) + + if (rc == 0) printf("PASSED"); + printf("\n"); munmap(ci0, bufsize); munmap(ci1, bufsize); free(mem0); free(mem1); + close(fd); return rc; } +static bool can_open_fb0(void) +{ + int fd; + + fd = open("/dev/fb0", O_RDWR); + if (fd < 0) + return false; + + close(fd); + return true; +} + int test_alignment_handler_vsx_206(void) { int rc = 0; + SKIP_IF(!can_open_fb0()); + SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06)); + printf("VSX: 2.06B\n"); LOAD_VSX_XFORM_TEST(lxvd2x); LOAD_VSX_XFORM_TEST(lxvw4x); @@ -285,6 +308,9 @@ int test_alignment_handler_vsx_207(void) { int rc = 0; + SKIP_IF(!can_open_fb0()); + SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_2_07)); + printf("VSX: 2.07B\n"); LOAD_VSX_XFORM_TEST(lxsspx); LOAD_VSX_XFORM_TEST(lxsiwax); @@ -298,6 +324,8 @@ int test_alignment_handler_vsx_300(void) { int rc = 0; + SKIP_IF(!can_open_fb0()); + SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_3_00)); printf("VSX: 3.00B\n"); LOAD_VMX_DFORM_TEST(lxsd); @@ -328,6 +356,8 @@ int test_alignment_handler_integer(void) { int rc = 0; + SKIP_IF(!can_open_fb0()); + printf("Integer\n"); LOAD_DFORM_TEST(lbz); LOAD_DFORM_TEST(lbzu); @@ -354,7 +384,6 @@ int test_alignment_handler_integer(void) LOAD_DFORM_TEST(ldu); LOAD_XFORM_TEST(ldx); LOAD_XFORM_TEST(ldux); - LOAD_XFORM_TEST(ldbrx); LOAD_DFORM_TEST(lmw); STORE_DFORM_TEST(stb); STORE_XFORM_TEST(stbx); @@ -374,8 +403,23 @@ int test_alignment_handler_integer(void) STORE_XFORM_TEST(stdx); STORE_DFORM_TEST(stdu); STORE_XFORM_TEST(stdux); - STORE_XFORM_TEST(stdbrx); STORE_DFORM_TEST(stmw); + + return rc; +} + +int test_alignment_handler_integer_206(void) +{ + int rc = 0; + + SKIP_IF(!can_open_fb0()); + SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06)); + + printf("Integer: 2.06\n"); + + LOAD_XFORM_TEST(ldbrx); + STORE_XFORM_TEST(stdbrx); + return rc; } @@ -383,6 +427,9 @@ int test_alignment_handler_vmx(void) { int rc = 0; + SKIP_IF(!can_open_fb0()); + SKIP_IF(!have_hwcap(PPC_FEATURE_HAS_ALTIVEC)); + printf("VMX\n"); LOAD_VMX_XFORM_TEST(lvx); @@ -408,23 +455,19 @@ int test_alignment_handler_fp(void) { int rc = 0; + SKIP_IF(!can_open_fb0()); + printf("Floating point\n"); LOAD_FLOAT_DFORM_TEST(lfd); LOAD_FLOAT_XFORM_TEST(lfdx); - LOAD_FLOAT_DFORM_TEST(lfdp); - LOAD_FLOAT_XFORM_TEST(lfdpx); LOAD_FLOAT_DFORM_TEST(lfdu); LOAD_FLOAT_XFORM_TEST(lfdux); LOAD_FLOAT_DFORM_TEST(lfs); LOAD_FLOAT_XFORM_TEST(lfsx); LOAD_FLOAT_DFORM_TEST(lfsu); LOAD_FLOAT_XFORM_TEST(lfsux); - LOAD_FLOAT_XFORM_TEST(lfiwzx); - LOAD_FLOAT_XFORM_TEST(lfiwax); STORE_FLOAT_DFORM_TEST(stfd); STORE_FLOAT_XFORM_TEST(stfdx); - STORE_FLOAT_DFORM_TEST(stfdp); - STORE_FLOAT_XFORM_TEST(stfdpx); STORE_FLOAT_DFORM_TEST(stfdu); STORE_FLOAT_XFORM_TEST(stfdux); STORE_FLOAT_DFORM_TEST(stfs); @@ -436,6 +479,38 @@ int test_alignment_handler_fp(void) return rc; } +int test_alignment_handler_fp_205(void) +{ + int rc = 0; + + SKIP_IF(!can_open_fb0()); + SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_05)); + + printf("Floating point: 2.05\n"); + + LOAD_FLOAT_DFORM_TEST(lfdp); + LOAD_FLOAT_XFORM_TEST(lfdpx); + LOAD_FLOAT_XFORM_TEST(lfiwax); + STORE_FLOAT_DFORM_TEST(stfdp); + STORE_FLOAT_XFORM_TEST(stfdpx); + + return rc; +} + +int test_alignment_handler_fp_206(void) +{ + int rc = 0; + + SKIP_IF(!can_open_fb0()); + SKIP_IF(!have_hwcap(PPC_FEATURE_ARCH_2_06)); + + printf("Floating point: 2.06\n"); + + LOAD_FLOAT_XFORM_TEST(lfiwzx); + + return rc; +} + void usage(char *prog) { printf("Usage: %s [options]\n", prog); @@ -483,9 +558,15 @@ int main(int argc, char *argv[]) "test_alignment_handler_vsx_300"); rc |= test_harness(test_alignment_handler_integer, "test_alignment_handler_integer"); + rc |= test_harness(test_alignment_handler_integer_206, + "test_alignment_handler_integer_206"); rc |= test_harness(test_alignment_handler_vmx, "test_alignment_handler_vmx"); rc |= test_harness(test_alignment_handler_fp, "test_alignment_handler_fp"); + rc |= test_harness(test_alignment_handler_fp_205, + "test_alignment_handler_fp_205"); + rc |= test_harness(test_alignment_handler_fp_206, + "test_alignment_handler_fp_206"); return rc; } diff --git a/tools/testing/selftests/powerpc/alignment/copy_first_unaligned.c b/tools/testing/selftests/powerpc/alignment/copy_first_unaligned.c index 47b73b3a08bd..5a9589987702 100644 --- a/tools/testing/selftests/powerpc/alignment/copy_first_unaligned.c +++ b/tools/testing/selftests/powerpc/alignment/copy_first_unaligned.c @@ -11,15 +11,46 @@ * */ +#include <signal.h> #include <string.h> #include <unistd.h> #include "utils.h" #include "instructions.h" -#include "copy_paste_unaligned_common.h" unsigned int expected_instruction = PPC_INST_COPY_FIRST; unsigned int instruction_mask = 0xfc2007fe; +void signal_action_handler(int signal_num, siginfo_t *info, void *ptr) +{ + ucontext_t *ctx = ptr; +#ifdef __powerpc64__ + unsigned int *pc = (unsigned int *)ctx->uc_mcontext.gp_regs[PT_NIP]; +#else + unsigned int *pc = (unsigned int *)ctx->uc_mcontext.uc_regs->gregs[PT_NIP]; +#endif + + /* + * Check that the signal was on the correct instruction, using a + * mask because the compiler assigns the register at RB. + */ + if ((*pc & instruction_mask) == expected_instruction) + _exit(0); /* We hit the right instruction */ + + _exit(1); +} + +void setup_signal_handler(void) +{ + struct sigaction signal_action; + + memset(&signal_action, 0, sizeof(signal_action)); + signal_action.sa_sigaction = signal_action_handler; + signal_action.sa_flags = SA_SIGINFO; + sigaction(SIGBUS, &signal_action, NULL); +} + +char cacheline_buf[128] __cacheline_aligned; + int test_copy_first_unaligned(void) { /* Only run this test on a P9 or later */ diff --git a/tools/testing/selftests/powerpc/alignment/copy_paste_unaligned_common.c b/tools/testing/selftests/powerpc/alignment/copy_paste_unaligned_common.c deleted file mode 100644 index d35fa5f5d2d3..000000000000 --- a/tools/testing/selftests/powerpc/alignment/copy_paste_unaligned_common.c +++ /dev/null @@ -1,53 +0,0 @@ -/* - * Copyright 2016, Chris Smart, IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * - * Common code for copy, copy_first, paste and paste_last unaligned - * tests. - * - */ - -#include <signal.h> -#include <string.h> -#include <unistd.h> -#include "utils.h" -#include "instructions.h" -#include "copy_paste_unaligned_common.h" - -unsigned int expected_instruction; -unsigned int instruction_mask; - -char cacheline_buf[128] __cacheline_aligned; - -void signal_action_handler(int signal_num, siginfo_t *info, void *ptr) -{ - ucontext_t *ctx = ptr; -#if defined(__powerpc64__) - unsigned int *pc = (unsigned int *)ctx->uc_mcontext.gp_regs[PT_NIP]; -#else - unsigned int *pc = (unsigned int *)ctx->uc_mcontext.uc_regs->gregs[PT_NIP]; -#endif - - /* - * Check that the signal was on the correct instruction, using a - * mask because the compiler assigns the register at RB. - */ - if ((*pc & instruction_mask) == expected_instruction) - _exit(0); /* We hit the right instruction */ - - _exit(1); -} - -void setup_signal_handler(void) -{ - struct sigaction signal_action; - - memset(&signal_action, 0, sizeof(signal_action)); - signal_action.sa_sigaction = signal_action_handler; - signal_action.sa_flags = SA_SIGINFO; - sigaction(SIGBUS, &signal_action, NULL); -} diff --git a/tools/testing/selftests/powerpc/alignment/copy_paste_unaligned_common.h b/tools/testing/selftests/powerpc/alignment/copy_paste_unaligned_common.h deleted file mode 100644 index 053899fe506e..000000000000 --- a/tools/testing/selftests/powerpc/alignment/copy_paste_unaligned_common.h +++ /dev/null @@ -1,26 +0,0 @@ -/* - * Copyright 2016, Chris Smart, IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * - * Declarations for common code for copy, copy_first, paste and - * paste_last unaligned tests. - * - */ - -#ifndef _SELFTESTS_POWERPC_COPY_PASTE_H -#define _SELFTESTS_POWERPC_COPY_PASTE_H - -#include <signal.h> - -int main(int argc, char *argv[]); -void signal_action_handler(int signal_num, siginfo_t *info, void *ptr); -void setup_signal_handler(void); -extern char cacheline_buf[128] __cacheline_aligned; -extern unsigned int expected_instruction; -extern unsigned int instruction_mask; - -#endif /* _SELFTESTS_POWERPC_COPY_PASTE_H */ diff --git a/tools/testing/selftests/powerpc/alignment/copy_unaligned.c b/tools/testing/selftests/powerpc/alignment/copy_unaligned.c deleted file mode 100644 index 3a4e26461554..000000000000 --- a/tools/testing/selftests/powerpc/alignment/copy_unaligned.c +++ /dev/null @@ -1,41 +0,0 @@ -/* - * Copyright 2016, Chris Smart, IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * - * Calls to copy which are not 128-byte aligned should be caught - * and sent a SIGBUS. - * - */ - -#include <string.h> -#include <unistd.h> -#include "utils.h" -#include "instructions.h" -#include "copy_paste_unaligned_common.h" - -unsigned int expected_instruction = PPC_INST_COPY; -unsigned int instruction_mask = 0xfc0007fe; - -int test_copy_unaligned(void) -{ - /* Only run this test on a P9 or later */ - SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_3_00)); - - /* Register our signal handler with SIGBUS */ - setup_signal_handler(); - - /* +1 makes buf unaligned */ - copy(cacheline_buf+1); - - /* We should not get here */ - return 1; -} - -int main(int argc, char *argv[]) -{ - return test_harness(test_copy_unaligned, "test_copy_unaligned"); -} diff --git a/tools/testing/selftests/powerpc/alignment/paste_last_unaligned.c b/tools/testing/selftests/powerpc/alignment/paste_last_unaligned.c deleted file mode 100644 index 6e0ad045fcc3..000000000000 --- a/tools/testing/selftests/powerpc/alignment/paste_last_unaligned.c +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2016, Chris Smart, IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * - * Calls to paste_last which are not 128-byte aligned should be - * caught and sent a SIGBUS. - * - */ - -#include <string.h> -#include <unistd.h> -#include "utils.h" -#include "instructions.h" -#include "copy_paste_unaligned_common.h" - -unsigned int expected_instruction = PPC_INST_PASTE_LAST; -unsigned int instruction_mask = 0xfc2007ff; - -int test_paste_last_unaligned(void) -{ - /* Only run this test on a P9 or later */ - SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_3_00)); - - /* Register our signal handler with SIGBUS */ - setup_signal_handler(); - - copy(cacheline_buf); - - /* +1 makes buf unaligned */ - paste_last(cacheline_buf+1); - - /* We should not get here */ - return 1; -} - -int main(int argc, char *argv[]) -{ - return test_harness(test_paste_last_unaligned, "test_paste_last_unaligned"); -} diff --git a/tools/testing/selftests/powerpc/alignment/paste_unaligned.c b/tools/testing/selftests/powerpc/alignment/paste_unaligned.c deleted file mode 100644 index 6f982b45e4bd..000000000000 --- a/tools/testing/selftests/powerpc/alignment/paste_unaligned.c +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright 2016, Chris Smart, IBM Corporation. - * - * This program is free software; you can redistribute it and/or - * modify it under the terms of the GNU General Public License - * as published by the Free Software Foundation; either version - * 2 of the License, or (at your option) any later version. - * - * Calls to paste which are not 128-byte aligned should be caught - * and sent a SIGBUS. - * - */ - -#include <string.h> -#include <unistd.h> -#include "utils.h" -#include "instructions.h" -#include "copy_paste_unaligned_common.h" - -unsigned int expected_instruction = PPC_INST_PASTE; -unsigned int instruction_mask = 0xfc0007fe; - -int test_paste_unaligned(void) -{ - /* Only run this test on a P9 or later */ - SKIP_IF(!have_hwcap2(PPC_FEATURE2_ARCH_3_00)); - - /* Register our signal handler with SIGBUS */ - setup_signal_handler(); - - copy(cacheline_buf); - - /* +1 makes buf unaligned */ - paste(cacheline_buf+1); - - /* We should not get here */ - return 1; -} - -int main(int argc, char *argv[]) -{ - return test_harness(test_paste_unaligned, "test_paste_unaligned"); -} diff --git a/tools/testing/selftests/powerpc/benchmarks/futex_bench.c b/tools/testing/selftests/powerpc/benchmarks/futex_bench.c index 2fc711d9150d..d58e4dc50fcd 100644 --- a/tools/testing/selftests/powerpc/benchmarks/futex_bench.c +++ b/tools/testing/selftests/powerpc/benchmarks/futex_bench.c @@ -38,5 +38,6 @@ int test_futex(void) int main(void) { + test_harness_set_timeout(300); return test_harness(test_futex, "futex_bench"); } diff --git a/tools/testing/selftests/powerpc/benchmarks/mmap_bench.c b/tools/testing/selftests/powerpc/benchmarks/mmap_bench.c index 7a0a462a2272..033de0560d99 100644 --- a/tools/testing/selftests/powerpc/benchmarks/mmap_bench.c +++ b/tools/testing/selftests/powerpc/benchmarks/mmap_bench.c @@ -84,5 +84,7 @@ int main(int argc, char *argv[]) exit(1); } } + + test_harness_set_timeout(300); return test_harness(test_mmap, "mmap_bench"); } diff --git a/tools/testing/selftests/powerpc/copyloops/.gitignore b/tools/testing/selftests/powerpc/copyloops/.gitignore index 25a192f62c4d..ce12cd0e2967 100644 --- a/tools/testing/selftests/powerpc/copyloops/.gitignore +++ b/tools/testing/selftests/powerpc/copyloops/.gitignore @@ -1,4 +1,13 @@ -copyuser_64 -copyuser_power7 -memcpy_64 -memcpy_power7 +copyuser_64_t0 +copyuser_64_t1 +copyuser_64_t2 +copyuser_power7_t0 +copyuser_power7_t1 +memcpy_64_t0 +memcpy_64_t1 +memcpy_64_t2 +memcpy_power7_t0 +memcpy_power7_t1 +copyuser_64_exc_t0 +copyuser_64_exc_t1 +copyuser_64_exc_t2 diff --git a/tools/testing/selftests/powerpc/copyloops/Makefile b/tools/testing/selftests/powerpc/copyloops/Makefile index eedce3366f64..1cf89a34d97c 100644 --- a/tools/testing/selftests/powerpc/copyloops/Makefile +++ b/tools/testing/selftests/powerpc/copyloops/Makefile @@ -4,18 +4,49 @@ CFLAGS += -m64 CFLAGS += -I$(CURDIR) CFLAGS += -D SELFTEST CFLAGS += -maltivec +CFLAGS += -mcpu=power4 # Use our CFLAGS for the implicit .S rule & set the asm machine type ASFLAGS = $(CFLAGS) -Wa,-mpower4 -TEST_GEN_PROGS := copyuser_64 copyuser_power7 memcpy_64 memcpy_power7 -EXTRA_SOURCES := validate.c ../harness.c +TEST_GEN_PROGS := copyuser_64_t0 copyuser_64_t1 copyuser_64_t2 \ + copyuser_p7_t0 copyuser_p7_t1 \ + memcpy_64_t0 memcpy_64_t1 memcpy_64_t2 \ + memcpy_p7_t0 memcpy_p7_t1 \ + copyuser_64_exc_t0 copyuser_64_exc_t1 copyuser_64_exc_t2 + +EXTRA_SOURCES := validate.c ../harness.c stubs.S include ../../lib.mk -$(OUTPUT)/copyuser_64: CPPFLAGS += -D COPY_LOOP=test___copy_tofrom_user_base -$(OUTPUT)/copyuser_power7: CPPFLAGS += -D COPY_LOOP=test___copy_tofrom_user_power7 -$(OUTPUT)/memcpy_64: CPPFLAGS += -D COPY_LOOP=test_memcpy -$(OUTPUT)/memcpy_power7: CPPFLAGS += -D COPY_LOOP=test_memcpy_power7 +$(OUTPUT)/copyuser_64_t%: copyuser_64.S $(EXTRA_SOURCES) + $(CC) $(CPPFLAGS) $(CFLAGS) \ + -D COPY_LOOP=test___copy_tofrom_user_base \ + -D SELFTEST_CASE=$(subst copyuser_64_t,,$(notdir $@)) \ + -o $@ $^ + +$(OUTPUT)/copyuser_p7_t%: copyuser_power7.S $(EXTRA_SOURCES) + $(CC) $(CPPFLAGS) $(CFLAGS) \ + -D COPY_LOOP=test___copy_tofrom_user_power7 \ + -D SELFTEST_CASE=$(subst copyuser_p7_t,,$(notdir $@)) \ + -o $@ $^ + +# Strictly speaking, we only need the memcpy_64 test cases for big-endian +$(OUTPUT)/memcpy_64_t%: memcpy_64.S $(EXTRA_SOURCES) + $(CC) $(CPPFLAGS) $(CFLAGS) \ + -D COPY_LOOP=test_memcpy \ + -D SELFTEST_CASE=$(subst memcpy_64_t,,$(notdir $@)) \ + -o $@ $^ + +$(OUTPUT)/memcpy_p7_t%: memcpy_power7.S $(EXTRA_SOURCES) + $(CC) $(CPPFLAGS) $(CFLAGS) \ + -D COPY_LOOP=test_memcpy_power7 \ + -D SELFTEST_CASE=$(subst memcpy_p7_t,,$(notdir $@)) \ + -o $@ $^ -$(TEST_GEN_PROGS): $(EXTRA_SOURCES) +$(OUTPUT)/copyuser_64_exc_t%: copyuser_64.S exc_validate.c ../harness.c \ + copy_tofrom_user_reference.S stubs.S + $(CC) $(CPPFLAGS) $(CFLAGS) \ + -D COPY_LOOP=test___copy_tofrom_user_base \ + -D SELFTEST_CASE=$(subst copyuser_64_exc_t,,$(notdir $@)) \ + -o $@ $^ diff --git a/tools/testing/selftests/powerpc/copyloops/asm/asm-compat.h b/tools/testing/selftests/powerpc/copyloops/asm/asm-compat.h new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/tools/testing/selftests/powerpc/copyloops/asm/asm-compat.h diff --git a/tools/testing/selftests/powerpc/copyloops/asm/feature-fixups.h b/tools/testing/selftests/powerpc/copyloops/asm/feature-fixups.h new file mode 100644 index 000000000000..e69de29bb2d1 --- /dev/null +++ b/tools/testing/selftests/powerpc/copyloops/asm/feature-fixups.h diff --git a/tools/testing/selftests/powerpc/copyloops/asm/ppc_asm.h b/tools/testing/selftests/powerpc/copyloops/asm/ppc_asm.h index 5ffe04d802c9..0605df807593 100644 --- a/tools/testing/selftests/powerpc/copyloops/asm/ppc_asm.h +++ b/tools/testing/selftests/powerpc/copyloops/asm/ppc_asm.h @@ -1,4 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef __SELFTESTS_POWERPC_PPC_ASM_H +#define __SELFTESTS_POWERPC_PPC_ASM_H #include <ppc-asm.h> #define CONFIG_ALTIVEC @@ -26,34 +28,20 @@ #define PPC_MTOCRF(A, B) mtocrf A, B -#define EX_TABLE(x, y) +#define EX_TABLE(x, y) \ + .section __ex_table,"a"; \ + .8byte x, y; \ + .previous -FUNC_START(enter_vmx_usercopy) - li r3,1 - blr +#define BEGIN_FTR_SECTION .if test_feature +#define FTR_SECTION_ELSE .else +#define ALT_FTR_SECTION_END_IFCLR(x) .endif +#define ALT_FTR_SECTION_END_IFSET(x) .endif +#define ALT_FTR_SECTION_END(x, y) .endif +#define END_FTR_SECTION_IFCLR(x) .endif +#define END_FTR_SECTION_IFSET(x) .endif -FUNC_START(exit_vmx_usercopy) - li r3,0 - blr +/* Default to taking the first of any alternative feature sections */ +test_feature = 1 -FUNC_START(enter_vmx_copy) - li r3,1 - blr - -FUNC_START(exit_vmx_copy) - blr - -FUNC_START(memcpy_power7) - blr - -FUNC_START(__copy_tofrom_user_power7) - blr - -FUNC_START(__copy_tofrom_user_base) - blr - -#define BEGIN_FTR_SECTION -#define FTR_SECTION_ELSE -#define ALT_FTR_SECTION_END_IFCLR(x) -#define ALT_FTR_SECTION_END(x, y) -#define END_FTR_SECTION_IFCLR(x) +#endif /* __SELFTESTS_POWERPC_PPC_ASM_H */ diff --git a/tools/testing/selftests/powerpc/copyloops/copy_tofrom_user_reference.S b/tools/testing/selftests/powerpc/copyloops/copy_tofrom_user_reference.S new file mode 100644 index 000000000000..3363b86407d6 --- /dev/null +++ b/tools/testing/selftests/powerpc/copyloops/copy_tofrom_user_reference.S @@ -0,0 +1,24 @@ +#include <asm/ppc_asm.h> + +_GLOBAL(copy_tofrom_user_reference) + cmpdi r5,0 + beq 4f + + mtctr r5 + +1: lbz r6,0(r4) +2: stb r6,0(r3) + addi r3,r3,1 + addi r4,r4,1 + bdnz 1b + +3: mfctr r3 + blr + +4: mr r3,r5 + blr + +.section __ex_table,"a" + .llong 1b,3b + .llong 2b,3b +.text diff --git a/tools/testing/selftests/powerpc/copyloops/exc_validate.c b/tools/testing/selftests/powerpc/copyloops/exc_validate.c new file mode 100644 index 000000000000..c896ea9a763c --- /dev/null +++ b/tools/testing/selftests/powerpc/copyloops/exc_validate.c @@ -0,0 +1,124 @@ +#include <stdlib.h> +#include <string.h> +#include <stdio.h> +#include <signal.h> +#include <unistd.h> +#include <sys/mman.h> + +#include "utils.h" + +extern char __start___ex_table[]; +extern char __stop___ex_table[]; + +#if defined(__powerpc64__) +#define UCONTEXT_NIA(UC) (UC)->uc_mcontext.gp_regs[PT_NIP] +#elif defined(__powerpc__) +#define UCONTEXT_NIA(UC) (UC)->uc_mcontext.uc_regs->gregs[PT_NIP] +#else +#error implement UCONTEXT_NIA +#endif + +static void segv_handler(int signr, siginfo_t *info, void *ptr) +{ + ucontext_t *uc = (ucontext_t *)ptr; + unsigned long addr = (unsigned long)info->si_addr; + unsigned long *ip = &UCONTEXT_NIA(uc); + unsigned long *ex_p = (unsigned long *)__start___ex_table; + + while (ex_p < (unsigned long *)__stop___ex_table) { + unsigned long insn, fixup; + + insn = *ex_p++; + fixup = *ex_p++; + + if (insn == *ip) { + *ip = fixup; + return; + } + } + + printf("No exception table match for NIA %lx ADDR %lx\n", *ip, addr); + abort(); +} + +static void setup_segv_handler(void) +{ + struct sigaction action; + + memset(&action, 0, sizeof(action)); + action.sa_sigaction = segv_handler; + action.sa_flags = SA_SIGINFO; + sigaction(SIGSEGV, &action, NULL); +} + +unsigned long COPY_LOOP(void *to, const void *from, unsigned long size); +unsigned long test_copy_tofrom_user_reference(void *to, const void *from, unsigned long size); + +static int total_passed; +static int total_failed; + +static void do_one_test(char *dstp, char *srcp, unsigned long len) +{ + unsigned long got, expected; + + got = COPY_LOOP(dstp, srcp, len); + expected = test_copy_tofrom_user_reference(dstp, srcp, len); + + if (got != expected) { + total_failed++; + printf("FAIL from=%p to=%p len=%ld returned %ld, expected %ld\n", + srcp, dstp, len, got, expected); + //abort(); + } else + total_passed++; +} + +//#define MAX_LEN 512 +#define MAX_LEN 16 + +int test_copy_exception(void) +{ + int page_size; + static char *p, *q; + unsigned long src, dst, len; + + page_size = getpagesize(); + p = mmap(NULL, page_size * 2, PROT_READ|PROT_WRITE, + MAP_PRIVATE|MAP_ANONYMOUS, -1, 0); + + if (p == MAP_FAILED) { + perror("mmap"); + exit(1); + } + + memset(p, 0, page_size); + + setup_segv_handler(); + + if (mprotect(p + page_size, page_size, PROT_NONE)) { + perror("mprotect"); + exit(1); + } + + q = p + page_size - MAX_LEN; + + for (src = 0; src < MAX_LEN; src++) { + for (dst = 0; dst < MAX_LEN; dst++) { + for (len = 0; len < MAX_LEN+1; len++) { + // printf("from=%p to=%p len=%ld\n", q+dst, q+src, len); + do_one_test(q+dst, q+src, len); + } + } + } + + printf("Totals:\n"); + printf(" Pass: %d\n", total_passed); + printf(" Fail: %d\n", total_failed); + + return 0; +} + +int main(void) +{ + return test_harness(test_copy_exception, str(COPY_LOOP)); +} diff --git a/tools/testing/selftests/powerpc/copyloops/stubs.S b/tools/testing/selftests/powerpc/copyloops/stubs.S new file mode 100644 index 000000000000..ec8bcf2bf1c2 --- /dev/null +++ b/tools/testing/selftests/powerpc/copyloops/stubs.S @@ -0,0 +1,19 @@ +#include <asm/ppc_asm.h> + +FUNC_START(enter_vmx_usercopy) + li r3,1 + blr + +FUNC_START(exit_vmx_usercopy) + li r3,0 + blr + +FUNC_START(enter_vmx_ops) + li r3,1 + blr + +FUNC_START(exit_vmx_ops) + blr + +FUNC_START(__copy_tofrom_user_base) + blr diff --git a/tools/testing/selftests/powerpc/dscr/dscr_inherit_exec_test.c b/tools/testing/selftests/powerpc/dscr/dscr_inherit_exec_test.c index 08a8b95e3bc1..c8c240accc0c 100644 --- a/tools/testing/selftests/powerpc/dscr/dscr_inherit_exec_test.c +++ b/tools/testing/selftests/powerpc/dscr/dscr_inherit_exec_test.c @@ -5,8 +5,8 @@ * verifies that the child is using the changed DSCR using mfspr. * * When using the privilege state SPR, the instructions such as - * mfspr or mtspr are priviledged and the kernel emulates them - * for us. Instructions using problem state SPR can be exuecuted + * mfspr or mtspr are privileged and the kernel emulates them + * for us. Instructions using problem state SPR can be executed * directly without any emulation if the HW supports them. Else * they also get emulated by the kernel. * @@ -19,7 +19,7 @@ */ #include "dscr.h" -static char prog[LEN_MAX]; +static char *prog; static void do_exec(unsigned long parent_dscr) { @@ -104,6 +104,6 @@ int main(int argc, char *argv[]) exit(1); } - strncpy(prog, argv[0], strlen(argv[0])); + prog = argv[0]; return test_harness(dscr_inherit_exec, "dscr_inherit_exec_test"); } diff --git a/tools/testing/selftests/powerpc/harness.c b/tools/testing/selftests/powerpc/harness.c index 66d31de60b9a..9d7166dfad1e 100644 --- a/tools/testing/selftests/powerpc/harness.c +++ b/tools/testing/selftests/powerpc/harness.c @@ -85,13 +85,13 @@ wait: return status; } -static void alarm_handler(int signum) +static void sig_handler(int signum) { - /* Jut wake us up from waitpid */ + /* Just wake us up from waitpid */ } -static struct sigaction alarm_action = { - .sa_handler = alarm_handler, +static struct sigaction sig_action = { + .sa_handler = sig_handler, }; void test_harness_set_timeout(uint64_t time) @@ -106,8 +106,14 @@ int test_harness(int (test_function)(void), char *name) test_start(name); test_set_git_version(GIT_VERSION); - if (sigaction(SIGALRM, &alarm_action, NULL)) { - perror("sigaction"); + if (sigaction(SIGINT, &sig_action, NULL)) { + perror("sigaction (sigint)"); + test_error(name); + return 1; + } + + if (sigaction(SIGALRM, &sig_action, NULL)) { + perror("sigaction (sigalrm)"); test_error(name); return 1; } diff --git a/tools/testing/selftests/powerpc/include/utils.h b/tools/testing/selftests/powerpc/include/utils.h index 735815b3ad7f..c58c370828b4 100644 --- a/tools/testing/selftests/powerpc/include/utils.h +++ b/tools/testing/selftests/powerpc/include/utils.h @@ -48,6 +48,8 @@ static inline bool have_hwcap2(unsigned long ftr2) } #endif +bool is_ppc64le(void); + /* Yes, this is evil */ #define FAIL_IF(x) \ do { \ diff --git a/tools/testing/selftests/powerpc/pmu/ebb/instruction_count_test.c b/tools/testing/selftests/powerpc/pmu/ebb/instruction_count_test.c index ae9a79086111..35a3426e341c 100644 --- a/tools/testing/selftests/powerpc/pmu/ebb/instruction_count_test.c +++ b/tools/testing/selftests/powerpc/pmu/ebb/instruction_count_test.c @@ -162,5 +162,6 @@ int instruction_count(void) int main(void) { + test_harness_set_timeout(300); return test_harness(instruction_count, "instruction_count"); } diff --git a/tools/testing/selftests/powerpc/pmu/ebb/lost_exception_test.c b/tools/testing/selftests/powerpc/pmu/ebb/lost_exception_test.c index eb8acb78bc6c..2ed7ad33f7a3 100644 --- a/tools/testing/selftests/powerpc/pmu/ebb/lost_exception_test.c +++ b/tools/testing/selftests/powerpc/pmu/ebb/lost_exception_test.c @@ -98,5 +98,6 @@ static int lost_exception(void) int main(void) { + test_harness_set_timeout(300); return test_harness(lost_exception, "lost_exception"); } diff --git a/tools/testing/selftests/powerpc/primitives/asm/asm-const.h b/tools/testing/selftests/powerpc/primitives/asm/asm-const.h new file mode 120000 index 000000000000..18d8be13e67f --- /dev/null +++ b/tools/testing/selftests/powerpc/primitives/asm/asm-const.h @@ -0,0 +1 @@ +../../../../../../arch/powerpc/include/asm/asm-const.h
\ No newline at end of file diff --git a/tools/testing/selftests/powerpc/primitives/asm/feature-fixups.h b/tools/testing/selftests/powerpc/primitives/asm/feature-fixups.h new file mode 120000 index 000000000000..8dc6d4d46e8e --- /dev/null +++ b/tools/testing/selftests/powerpc/primitives/asm/feature-fixups.h @@ -0,0 +1 @@ +../../../../../../arch/powerpc/include/asm/feature-fixups.h
\ No newline at end of file diff --git a/tools/testing/selftests/powerpc/ptrace/core-pkey.c b/tools/testing/selftests/powerpc/ptrace/core-pkey.c index 36bc312b1f5c..e23e2e199eb4 100644 --- a/tools/testing/selftests/powerpc/ptrace/core-pkey.c +++ b/tools/testing/selftests/powerpc/ptrace/core-pkey.c @@ -140,6 +140,10 @@ static int child(struct shared_info *info) if (disable_execute) info->iamr |= 1ul << pkeyshift(pkey1); + else + info->iamr &= ~(1ul << pkeyshift(pkey1)); + + info->iamr &= ~(1ul << pkeyshift(pkey2) | 1ul << pkeyshift(pkey3)); info->uamor |= 3ul << pkeyshift(pkey1) | 3ul << pkeyshift(pkey2); diff --git a/tools/testing/selftests/powerpc/ptrace/ptrace-pkey.c b/tools/testing/selftests/powerpc/ptrace/ptrace-pkey.c index 5cf631f792cc..bdbbbe8431e0 100644 --- a/tools/testing/selftests/powerpc/ptrace/ptrace-pkey.c +++ b/tools/testing/selftests/powerpc/ptrace/ptrace-pkey.c @@ -104,6 +104,10 @@ static int child(struct shared_info *info) if (disable_execute) info->expected_iamr |= 1ul << pkeyshift(pkey1); + else + info->expected_iamr &= ~(1ul << pkeyshift(pkey1)); + + info->expected_iamr &= ~(1ul << pkeyshift(pkey2) | 1ul << pkeyshift(pkey3)); info->expected_uamor |= 3ul << pkeyshift(pkey1) | 3ul << pkeyshift(pkey2); diff --git a/tools/testing/selftests/powerpc/stringloops/Makefile b/tools/testing/selftests/powerpc/stringloops/Makefile index 1125e489055e..10b35c87a4f4 100644 --- a/tools/testing/selftests/powerpc/stringloops/Makefile +++ b/tools/testing/selftests/powerpc/stringloops/Makefile @@ -1,10 +1,33 @@ # SPDX-License-Identifier: GPL-2.0 # The loops are all 64-bit code -CFLAGS += -m64 CFLAGS += -I$(CURDIR) -TEST_GEN_PROGS := memcmp -EXTRA_SOURCES := memcmp_64.S ../harness.c +EXTRA_SOURCES := ../harness.c + +build_32bit = $(shell if ($(CC) $(CFLAGS) -m32 -o /dev/null memcmp.c >/dev/null 2>&1) then echo "1"; fi) + +TEST_GEN_PROGS := memcmp_64 strlen + +$(OUTPUT)/memcmp_64: memcmp.c +$(OUTPUT)/memcmp_64: CFLAGS += -m64 -maltivec + +ifeq ($(build_32bit),1) +$(OUTPUT)/memcmp_32: memcmp.c +$(OUTPUT)/memcmp_32: CFLAGS += -m32 + +TEST_GEN_PROGS += memcmp_32 +endif + +$(OUTPUT)/strlen: strlen.c string.c + +ifeq ($(build_32bit),1) +$(OUTPUT)/strlen_32: strlen.c +$(OUTPUT)/strlen_32: CFLAGS += -m32 + +TEST_GEN_PROGS += strlen_32 +endif + +ASFLAGS = $(CFLAGS) include ../../lib.mk diff --git a/tools/testing/selftests/powerpc/stringloops/asm/cache.h b/tools/testing/selftests/powerpc/stringloops/asm/cache.h new file mode 100644 index 000000000000..8a2840831122 --- /dev/null +++ b/tools/testing/selftests/powerpc/stringloops/asm/cache.h @@ -0,0 +1 @@ +#define IFETCH_ALIGN_BYTES 4 diff --git a/tools/testing/selftests/powerpc/stringloops/asm/ppc-opcode.h b/tools/testing/selftests/powerpc/stringloops/asm/ppc-opcode.h new file mode 100644 index 000000000000..9de413c0c2cb --- /dev/null +++ b/tools/testing/selftests/powerpc/stringloops/asm/ppc-opcode.h @@ -0,0 +1,39 @@ +/* + * Copyright 2009 Freescale Semiconductor, Inc. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + * + * provides masks and opcode images for use by code generation, emulation + * and for instructions that older assemblers might not know about + */ +#ifndef _ASM_POWERPC_PPC_OPCODE_H +#define _ASM_POWERPC_PPC_OPCODE_H + + +# define stringify_in_c(...) __VA_ARGS__ +# define ASM_CONST(x) x + + +#define PPC_INST_VCMPEQUD_RC 0x100000c7 +#define PPC_INST_VCMPEQUB_RC 0x10000006 + +#define __PPC_RC21 (0x1 << 10) + +/* macros to insert fields into opcodes */ +#define ___PPC_RA(a) (((a) & 0x1f) << 16) +#define ___PPC_RB(b) (((b) & 0x1f) << 11) +#define ___PPC_RS(s) (((s) & 0x1f) << 21) +#define ___PPC_RT(t) ___PPC_RS(t) + +#define VCMPEQUD_RC(vrt, vra, vrb) stringify_in_c(.long PPC_INST_VCMPEQUD_RC | \ + ___PPC_RT(vrt) | ___PPC_RA(vra) | \ + ___PPC_RB(vrb) | __PPC_RC21) + +#define VCMPEQUB_RC(vrt, vra, vrb) stringify_in_c(.long PPC_INST_VCMPEQUB_RC | \ + ___PPC_RT(vrt) | ___PPC_RA(vra) | \ + ___PPC_RB(vrb) | __PPC_RC21) + +#endif /* _ASM_POWERPC_PPC_OPCODE_H */ diff --git a/tools/testing/selftests/powerpc/stringloops/asm/ppc_asm.h b/tools/testing/selftests/powerpc/stringloops/asm/ppc_asm.h index 136242ec4b0e..d2c0a911f55e 100644 --- a/tools/testing/selftests/powerpc/stringloops/asm/ppc_asm.h +++ b/tools/testing/selftests/powerpc/stringloops/asm/ppc_asm.h @@ -1,4 +1,6 @@ /* SPDX-License-Identifier: GPL-2.0 */ +#ifndef _PPC_ASM_H +#define __PPC_ASM_H #include <ppc-asm.h> #ifndef r1 @@ -6,3 +8,26 @@ #endif #define _GLOBAL(A) FUNC_START(test_ ## A) +#define _GLOBAL_TOC(A) FUNC_START(test_ ## A) + +#define CONFIG_ALTIVEC + +#define R14 r14 +#define R15 r15 +#define R16 r16 +#define R17 r17 +#define R18 r18 +#define R19 r19 +#define R20 r20 +#define R21 r21 +#define R22 r22 +#define R29 r29 +#define R30 r30 +#define R31 r31 + +#define STACKFRAMESIZE 256 +#define STK_REG(i) (112 + ((i)-14)*8) + +#define BEGIN_FTR_SECTION +#define END_FTR_SECTION_IFSET(val) +#endif diff --git a/tools/testing/selftests/powerpc/stringloops/memcmp.c b/tools/testing/selftests/powerpc/stringloops/memcmp.c index 8250db25b379..b1fa7546957f 100644 --- a/tools/testing/selftests/powerpc/stringloops/memcmp.c +++ b/tools/testing/selftests/powerpc/stringloops/memcmp.c @@ -2,20 +2,40 @@ #include <malloc.h> #include <stdlib.h> #include <string.h> +#include <time.h> #include "utils.h" #define SIZE 256 #define ITERATIONS 10000 +#define LARGE_SIZE (5 * 1024) +#define LARGE_ITERATIONS 1000 +#define LARGE_MAX_OFFSET 32 +#define LARGE_SIZE_START 4096 + +#define MAX_OFFSET_DIFF_S1_S2 48 + +int vmx_count; +int enter_vmx_ops(void) +{ + vmx_count++; + return 1; +} + +void exit_vmx_ops(void) +{ + vmx_count--; +} int test_memcmp(const void *s1, const void *s2, size_t n); /* test all offsets and lengths */ -static void test_one(char *s1, char *s2) +static void test_one(char *s1, char *s2, unsigned long max_offset, + unsigned long size_start, unsigned long max_size) { unsigned long offset, size; - for (offset = 0; offset < SIZE; offset++) { - for (size = 0; size < (SIZE-offset); size++) { + for (offset = 0; offset < max_offset; offset++) { + for (size = size_start; size < (max_size - offset); size++) { int x, y; unsigned long i; @@ -35,70 +55,105 @@ static void test_one(char *s1, char *s2) printf("\n"); abort(); } + + if (vmx_count != 0) { + printf("vmx enter/exit not paired.(offset:%ld size:%ld s1:%p s2:%p vc:%d\n", + offset, size, s1, s2, vmx_count); + printf("\n"); + abort(); + } } } } -static int testcase(void) +static int testcase(bool islarge) { char *s1; char *s2; unsigned long i; - s1 = memalign(128, SIZE); + unsigned long comp_size = (islarge ? LARGE_SIZE : SIZE); + unsigned long alloc_size = comp_size + MAX_OFFSET_DIFF_S1_S2; + int iterations = islarge ? LARGE_ITERATIONS : ITERATIONS; + + s1 = memalign(128, alloc_size); if (!s1) { perror("memalign"); exit(1); } - s2 = memalign(128, SIZE); + s2 = memalign(128, alloc_size); if (!s2) { perror("memalign"); exit(1); } - srandom(1); + srandom(time(0)); - for (i = 0; i < ITERATIONS; i++) { + for (i = 0; i < iterations; i++) { unsigned long j; unsigned long change; + char *rand_s1 = s1; + char *rand_s2 = s2; - for (j = 0; j < SIZE; j++) + for (j = 0; j < alloc_size; j++) s1[j] = random(); - memcpy(s2, s1, SIZE); + rand_s1 += random() % MAX_OFFSET_DIFF_S1_S2; + rand_s2 += random() % MAX_OFFSET_DIFF_S1_S2; + memcpy(rand_s2, rand_s1, comp_size); /* change one byte */ - change = random() % SIZE; - s2[change] = random() & 0xff; - - test_one(s1, s2); + change = random() % comp_size; + rand_s2[change] = random() & 0xff; + + if (islarge) + test_one(rand_s1, rand_s2, LARGE_MAX_OFFSET, + LARGE_SIZE_START, comp_size); + else + test_one(rand_s1, rand_s2, SIZE, 0, comp_size); } - srandom(1); + srandom(time(0)); - for (i = 0; i < ITERATIONS; i++) { + for (i = 0; i < iterations; i++) { unsigned long j; unsigned long change; + char *rand_s1 = s1; + char *rand_s2 = s2; - for (j = 0; j < SIZE; j++) + for (j = 0; j < alloc_size; j++) s1[j] = random(); - memcpy(s2, s1, SIZE); + rand_s1 += random() % MAX_OFFSET_DIFF_S1_S2; + rand_s2 += random() % MAX_OFFSET_DIFF_S1_S2; + memcpy(rand_s2, rand_s1, comp_size); /* change multiple bytes, 1/8 of total */ - for (j = 0; j < SIZE / 8; j++) { - change = random() % SIZE; + for (j = 0; j < comp_size / 8; j++) { + change = random() % comp_size; s2[change] = random() & 0xff; } - test_one(s1, s2); + if (islarge) + test_one(rand_s1, rand_s2, LARGE_MAX_OFFSET, + LARGE_SIZE_START, comp_size); + else + test_one(rand_s1, rand_s2, SIZE, 0, comp_size); } return 0; } +static int testcases(void) +{ + testcase(0); + testcase(1); + return 0; +} + int main(void) { - return test_harness(testcase, "memcmp"); + test_harness_set_timeout(300); + return test_harness(testcases, "memcmp"); } diff --git a/tools/testing/selftests/powerpc/stringloops/memcmp_32.S b/tools/testing/selftests/powerpc/stringloops/memcmp_32.S new file mode 120000 index 000000000000..056f2b3af789 --- /dev/null +++ b/tools/testing/selftests/powerpc/stringloops/memcmp_32.S @@ -0,0 +1 @@ +../../../../../arch/powerpc/lib/memcmp_32.S
\ No newline at end of file diff --git a/tools/testing/selftests/powerpc/stringloops/string.c b/tools/testing/selftests/powerpc/stringloops/string.c new file mode 100644 index 000000000000..45e7775415c7 --- /dev/null +++ b/tools/testing/selftests/powerpc/stringloops/string.c @@ -0,0 +1,21 @@ +// SPDX-License-Identifier: GPL-2.0 +/* + * Copied from linux/lib/string.c + * + * Copyright (C) 1991, 1992 Linus Torvalds + */ + +#include <stddef.h> + +/** + * strlen - Find the length of a string + * @s: The string to be sized + */ +size_t test_strlen(const char *s) +{ + const char *sc; + + for (sc = s; *sc != '\0'; ++sc) + /* nothing */; + return sc - s; +} diff --git a/tools/testing/selftests/powerpc/stringloops/strlen.c b/tools/testing/selftests/powerpc/stringloops/strlen.c new file mode 100644 index 000000000000..9055ebc484d0 --- /dev/null +++ b/tools/testing/selftests/powerpc/stringloops/strlen.c @@ -0,0 +1,127 @@ +// SPDX-License-Identifier: GPL-2.0 +#include <malloc.h> +#include <stdlib.h> +#include <string.h> +#include <time.h> +#include "utils.h" + +#define SIZE 256 +#define ITERATIONS 1000 +#define ITERATIONS_BENCH 100000 + +int test_strlen(const void *s); + +/* test all offsets and lengths */ +static void test_one(char *s) +{ + unsigned long offset; + + for (offset = 0; offset < SIZE; offset++) { + int x, y; + unsigned long i; + + y = strlen(s + offset); + x = test_strlen(s + offset); + + if (x != y) { + printf("strlen() returned %d, should have returned %d (%p offset %ld)\n", x, y, s, offset); + + for (i = offset; i < SIZE; i++) + printf("%02x ", s[i]); + printf("\n"); + } + } +} + +static void bench_test(char *s) +{ + struct timespec ts_start, ts_end; + int i; + + clock_gettime(CLOCK_MONOTONIC, &ts_start); + + for (i = 0; i < ITERATIONS_BENCH; i++) + test_strlen(s); + + clock_gettime(CLOCK_MONOTONIC, &ts_end); + + printf("len %3.3d : time = %.6f\n", test_strlen(s), ts_end.tv_sec - ts_start.tv_sec + (ts_end.tv_nsec - ts_start.tv_nsec) / 1e9); +} + +static int testcase(void) +{ + char *s; + unsigned long i; + + s = memalign(128, SIZE); + if (!s) { + perror("memalign"); + exit(1); + } + + srandom(1); + + memset(s, 0, SIZE); + for (i = 0; i < SIZE; i++) { + char c; + + do { + c = random() & 0x7f; + } while (!c); + s[i] = c; + test_one(s); + } + + for (i = 0; i < ITERATIONS; i++) { + unsigned long j; + + for (j = 0; j < SIZE; j++) { + char c; + + do { + c = random() & 0x7f; + } while (!c); + s[j] = c; + } + for (j = 0; j < sizeof(long); j++) { + s[SIZE - 1 - j] = 0; + test_one(s); + } + } + + for (i = 0; i < SIZE; i++) { + char c; + + do { + c = random() & 0x7f; + } while (!c); + s[i] = c; + } + + bench_test(s); + + s[16] = 0; + bench_test(s); + + s[8] = 0; + bench_test(s); + + s[4] = 0; + bench_test(s); + + s[3] = 0; + bench_test(s); + + s[2] = 0; + bench_test(s); + + s[1] = 0; + bench_test(s); + + return 0; +} + +int main(void) +{ + return test_harness(testcase, "strlen"); +} diff --git a/tools/testing/selftests/powerpc/stringloops/strlen_32.S b/tools/testing/selftests/powerpc/stringloops/strlen_32.S new file mode 120000 index 000000000000..72b13731b24c --- /dev/null +++ b/tools/testing/selftests/powerpc/stringloops/strlen_32.S @@ -0,0 +1 @@ +../../../../../arch/powerpc/lib/strlen_32.S
\ No newline at end of file diff --git a/tools/testing/selftests/powerpc/tm/tm-sigreturn.c b/tools/testing/selftests/powerpc/tm/tm-sigreturn.c index 85d63449243b..9a6017a1d769 100644 --- a/tools/testing/selftests/powerpc/tm/tm-sigreturn.c +++ b/tools/testing/selftests/powerpc/tm/tm-sigreturn.c @@ -55,6 +55,7 @@ int tm_sigreturn(void) uint64_t ret = 0; SKIP_IF(!have_htm()); + SKIP_IF(!is_ppc64le()); memset(&sa, 0, sizeof(sa)); sa.sa_handler = handler; diff --git a/tools/testing/selftests/powerpc/tm/tm-tar.c b/tools/testing/selftests/powerpc/tm/tm-tar.c index 2d2fcc2b7a60..f31fe5a28ddb 100644 --- a/tools/testing/selftests/powerpc/tm/tm-tar.c +++ b/tools/testing/selftests/powerpc/tm/tm-tar.c @@ -26,6 +26,7 @@ int test_tar(void) int i; SKIP_IF(!have_htm()); + SKIP_IF(!is_ppc64le()); for (i = 0; i < num_loops; i++) { diff --git a/tools/testing/selftests/powerpc/tm/tm-vmxcopy.c b/tools/testing/selftests/powerpc/tm/tm-vmxcopy.c index 0274de7b11f3..fe52811584ae 100644 --- a/tools/testing/selftests/powerpc/tm/tm-vmxcopy.c +++ b/tools/testing/selftests/powerpc/tm/tm-vmxcopy.c @@ -46,6 +46,7 @@ int test_vmxcopy() uint64_t aborted = 0; SKIP_IF(!have_htm()); + SKIP_IF(!is_ppc64le()); fd = mkstemp(tmpfile); assert(fd >= 0); diff --git a/tools/testing/selftests/powerpc/utils.c b/tools/testing/selftests/powerpc/utils.c index d46916867a6f..aa8fc1e6365b 100644 --- a/tools/testing/selftests/powerpc/utils.c +++ b/tools/testing/selftests/powerpc/utils.c @@ -11,8 +11,10 @@ #include <link.h> #include <sched.h> #include <stdio.h> +#include <string.h> #include <sys/stat.h> #include <sys/types.h> +#include <sys/utsname.h> #include <unistd.h> #include "utils.h" @@ -104,3 +106,18 @@ int pick_online_cpu(void) printf("No cpus in affinity mask?!\n"); return -1; } + +bool is_ppc64le(void) +{ + struct utsname uts; + int rc; + + errno = 0; + rc = uname(&uts); + if (rc) { + perror("uname"); + return false; + } + + return strcmp(uts.machine, "ppc64le") == 0; +} |