diff options
| author | Dave Airlie <airlied@redhat.com> | 2018-03-23 06:16:51 +1000 |
|---|---|---|
| committer | Dave Airlie <airlied@redhat.com> | 2018-03-23 06:16:51 +1000 |
| commit | f3924ae723d84746546bd74bdefc99c17da2a467 (patch) | |
| tree | db57c3cddaab0db958de7f537e668267aae4115e /drivers/gpu | |
| parent | 0c5286a8222e1d178f57115401673ac34126a510 (diff) | |
| parent | 4ed75c3e525598ff1aa6aed09c419c73a4efd2f2 (diff) | |
Merge branch 'etnaviv/next' of https://git.pengutronix.de/git/lst/linux into drm-next
Changes this time mostly come down to:
- hook up the DRM GPU scheduler
- prep work for GC7000L support, to be completed in the next cycle
* 'etnaviv/next' of https://git.pengutronix.de/git/lst/linux: (22 commits)
drm/etnaviv: bump HW job limit to 4
drm/etnaviv: etnaviv_sched: Staticize functions when possible
drm/etnaviv: add PTA handling to MMUv2
drm/etnaviv: add function to load the initial PTA state
drm/etnaviv: handle security states
drm/etnaviv: add security handling mode enum
drm/etnaviv: add hardware database
drm/etnaviv: add more minor features fields
drm/etnaviv: update hardware headers from rnndb
drm/etnaviv: add support for slave interface clock
drm/etnaviv: split out and optimize MMU fault dumping
drm/etnaviv: remove the need for a gpu-subsystem DT node
dt-bindings: etnaviv: add slave interface clock
drm/etnaviv: use correct format specifier for size_t
drm/etnaviv: replace hangcheck with scheduler timeout
drm/etnaviv: lock BOs after all other submit work is done
drm/etnaviv: move dependency handling to scheduler
drm/etnaviv: hook up DRM GPU scheduler
drm/etnaviv: track fences by IDR instead of seqno
drm/etnaviv: add missing major features field to debugfs
...
Diffstat (limited to 'drivers/gpu')
21 files changed, 1301 insertions, 434 deletions
diff --git a/drivers/gpu/drm/etnaviv/Kconfig b/drivers/gpu/drm/etnaviv/Kconfig index 3f58b4077767..e5bfeca361bd 100644 --- a/drivers/gpu/drm/etnaviv/Kconfig +++ b/drivers/gpu/drm/etnaviv/Kconfig @@ -11,6 +11,7 @@ config DRM_ETNAVIV select WANT_DEV_COREDUMP select CMA if HAVE_DMA_CONTIGUOUS select DMA_CMA if HAVE_DMA_CONTIGUOUS + select DRM_SCHED help DRM driver for Vivante GPUs. diff --git a/drivers/gpu/drm/etnaviv/Makefile b/drivers/gpu/drm/etnaviv/Makefile index 1281c8d4fae5..46e5ffad69a6 100644 --- a/drivers/gpu/drm/etnaviv/Makefile +++ b/drivers/gpu/drm/etnaviv/Makefile @@ -9,9 +9,11 @@ etnaviv-y := \ etnaviv_gem_submit.o \ etnaviv_gem.o \ etnaviv_gpu.o \ + etnaviv_hwdb.o \ etnaviv_iommu_v2.o \ etnaviv_iommu.o \ etnaviv_mmu.o \ - etnaviv_perfmon.o + etnaviv_perfmon.o \ + etnaviv_sched.o obj-$(CONFIG_DRM_ETNAVIV) += etnaviv.o diff --git a/drivers/gpu/drm/etnaviv/common.xml.h b/drivers/gpu/drm/etnaviv/common.xml.h index 207f45c999c3..001faea80fef 100644 --- a/drivers/gpu/drm/etnaviv/common.xml.h +++ b/drivers/gpu/drm/etnaviv/common.xml.h @@ -8,15 +8,12 @@ http://0x04.net/cgit/index.cgi/rules-ng-ng git clone git://0x04.net/rules-ng-ng The rules-ng-ng source files this header was generated from are: -- state.xml ( 19930 bytes, from 2017-03-09 15:43:43) -- common.xml ( 23473 bytes, from 2017-03-09 15:43:43) -- state_hi.xml ( 26403 bytes, from 2017-03-09 15:43:43) -- copyright.xml ( 1597 bytes, from 2016-12-08 16:37:56) -- state_2d.xml ( 51552 bytes, from 2016-12-08 16:37:56) -- state_3d.xml ( 66957 bytes, from 2017-03-09 15:43:43) -- state_vg.xml ( 5975 bytes, from 2016-12-08 16:37:56) +- texdesc_3d.xml ( 3183 bytes, from 2017-12-18 16:51:59) +- copyright.xml ( 1597 bytes, from 2016-12-08 16:37:56) +- common.xml ( 35468 bytes, from 2018-01-22 13:48:54) +- common_3d.xml ( 14615 bytes, from 2017-12-18 16:51:59) -Copyright (C) 2012-2017 by the following authors: +Copyright (C) 2012-2018 by the following authors: - Wladimir J. van der Laan <laanwj@gmail.com> - Christian Gmeiner <christian.gmeiner@gmail.com> - Lucas Stach <l.stach@pengutronix.de> @@ -49,12 +46,7 @@ DEALINGS IN THE SOFTWARE. #define SYNC_RECIPIENT_RA 0x00000005 #define SYNC_RECIPIENT_PE 0x00000007 #define SYNC_RECIPIENT_DE 0x0000000b -#define SYNC_RECIPIENT_VG 0x0000000f -#define SYNC_RECIPIENT_TESSELATOR 0x00000010 -#define SYNC_RECIPIENT_VG2 0x00000011 -#define SYNC_RECIPIENT_TESSELATOR2 0x00000012 -#define SYNC_RECIPIENT_VG3 0x00000013 -#define SYNC_RECIPIENT_TESSELATOR3 0x00000014 +#define SYNC_RECIPIENT_BLT 0x00000010 #define ENDIAN_MODE_NO_SWAP 0x00000000 #define ENDIAN_MODE_SWAP_16 0x00000001 #define ENDIAN_MODE_SWAP_32 0x00000002 @@ -77,6 +69,7 @@ DEALINGS IN THE SOFTWARE. #define chipModel_GC800 0x00000800 #define chipModel_GC860 0x00000860 #define chipModel_GC880 0x00000880 +#define chipModel_GC900 0x00000900 #define chipModel_GC1000 0x00001000 #define chipModel_GC1500 0x00001500 #define chipModel_GC2000 0x00002000 @@ -88,6 +81,12 @@ DEALINGS IN THE SOFTWARE. #define chipModel_GC5000 0x00005000 #define chipModel_GC5200 0x00005200 #define chipModel_GC6400 0x00006400 +#define chipModel_GC7000 0x00007000 +#define chipModel_GC7400 0x00007400 +#define chipModel_GC8000 0x00008000 +#define chipModel_GC8100 0x00008100 +#define chipModel_GC8200 0x00008200 +#define chipModel_GC8400 0x00008400 #define RGBA_BITS_R 0x00000001 #define RGBA_BITS_G 0x00000002 #define RGBA_BITS_B 0x00000004 @@ -203,7 +202,7 @@ DEALINGS IN THE SOFTWARE. #define chipMinorFeatures2_RGB888 0x00001000 #define chipMinorFeatures2_TX__YUV_ASSEMBLER 0x00002000 #define chipMinorFeatures2_DYNAMIC_FREQUENCY_SCALING 0x00004000 -#define chipMinorFeatures2_EXTRA_TEXTURE_STATE 0x00008000 +#define chipMinorFeatures2_TX_FILTER 0x00008000 #define chipMinorFeatures2_FULL_DIRECTFB 0x00010000 #define chipMinorFeatures2_2D_TILING 0x00020000 #define chipMinorFeatures2_THREAD_WALKER_IN_PS 0x00040000 @@ -242,36 +241,36 @@ DEALINGS IN THE SOFTWARE. #define chipMinorFeatures3_TX_ENHANCEMENTS1 0x00080000 #define chipMinorFeatures3_SH_ENHANCEMENTS1 0x00100000 #define chipMinorFeatures3_SH_ENHANCEMENTS2 0x00200000 -#define chipMinorFeatures3_UNK22 0x00400000 +#define chipMinorFeatures3_PE_ENHANCEMENTS1 0x00400000 #define chipMinorFeatures3_2D_FC_SOURCE 0x00800000 -#define chipMinorFeatures3_UNK24 0x01000000 -#define chipMinorFeatures3_UNK25 0x02000000 +#define chipMinorFeatures3_BUG_FIXES_14 0x01000000 +#define chipMinorFeatures3_POWER_OPTIMIZATIONS_0 0x02000000 #define chipMinorFeatures3_NEW_HZ 0x04000000 -#define chipMinorFeatures3_UNK27 0x08000000 -#define chipMinorFeatures3_UNK28 0x10000000 +#define chipMinorFeatures3_PE_DITHER_FIX 0x08000000 +#define chipMinorFeatures3_DE_ENHANCEMENTS3 0x10000000 #define chipMinorFeatures3_SH_ENHANCEMENTS3 0x20000000 -#define chipMinorFeatures3_UNK30 0x40000000 -#define chipMinorFeatures3_UNK31 0x80000000 -#define chipMinorFeatures4_UNK0 0x00000001 +#define chipMinorFeatures3_SH_ENHANCEMENTS4 0x40000000 +#define chipMinorFeatures3_TX_ENHANCEMENTS2 0x80000000 +#define chipMinorFeatures4_FE_ENHANCEMENTS1 0x00000001 #define chipMinorFeatures4_PE_ENHANCEMENTS2 0x00000002 #define chipMinorFeatures4_FRUSTUM_CLIP_FIX 0x00000004 -#define chipMinorFeatures4_UNK3 0x00000008 -#define chipMinorFeatures4_UNK4 0x00000010 +#define chipMinorFeatures4_DE_NO_GAMMA 0x00000008 +#define chipMinorFeatures4_PA_ENHANCEMENTS_2 0x00000010 #define chipMinorFeatures4_2D_GAMMA 0x00000020 #define chipMinorFeatures4_SINGLE_BUFFER 0x00000040 -#define chipMinorFeatures4_UNK7 0x00000080 -#define chipMinorFeatures4_UNK8 0x00000100 -#define chipMinorFeatures4_UNK9 0x00000200 -#define chipMinorFeatures4_UNK10 0x00000400 +#define chipMinorFeatures4_HI_ENHANCEMENTS_1 0x00000080 +#define chipMinorFeatures4_TX_ENHANCEMENTS_3 0x00000100 +#define chipMinorFeatures4_SH_ENHANCEMENTS_5 0x00000200 +#define chipMinorFeatures4_FE_ENHANCEMENTS_2 0x00000400 #define chipMinorFeatures4_TX_LERP_PRECISION_FIX 0x00000800 #define chipMinorFeatures4_2D_COLOR_SPACE_CONVERSION 0x00001000 #define chipMinorFeatures4_TEXTURE_ASTC 0x00002000 -#define chipMinorFeatures4_UNK14 0x00004000 -#define chipMinorFeatures4_UNK15 0x00008000 +#define chipMinorFeatures4_PE_ENHANCEMENTS_4 0x00004000 +#define chipMinorFeatures4_MC_ENHANCEMENTS_1 0x00008000 #define chipMinorFeatures4_HALTI2 0x00010000 -#define chipMinorFeatures4_UNK17 0x00020000 +#define chipMinorFeatures4_2D_MIRROR_EXTENSION 0x00020000 #define chipMinorFeatures4_SMALL_MSAA 0x00040000 -#define chipMinorFeatures4_UNK19 0x00080000 +#define chipMinorFeatures4_BUG_FIXES_17 0x00080000 #define chipMinorFeatures4_NEW_RA 0x00100000 #define chipMinorFeatures4_2D_OPF_YUV_OUTPUT 0x00200000 #define chipMinorFeatures4_2D_MULTI_SOURCE_BLT_EX2 0x00400000 @@ -280,41 +279,207 @@ DEALINGS IN THE SOFTWARE. #define chipMinorFeatures4_BUG_FIXES18 0x02000000 #define chipMinorFeatures4_2D_COMPRESSION 0x04000000 #define chipMinorFeatures4_PROBE 0x08000000 -#define chipMinorFeatures4_UNK28 0x10000000 +#define chipMinorFeatures4_MEDIUM_PRECISION 0x10000000 #define chipMinorFeatures4_2D_SUPER_TILE_VERSION 0x20000000 -#define chipMinorFeatures4_UNK30 0x40000000 -#define chipMinorFeatures4_UNK31 0x80000000 -#define chipMinorFeatures5_UNK0 0x00000001 -#define chipMinorFeatures5_UNK1 0x00000002 -#define chipMinorFeatures5_UNK2 0x00000004 -#define chipMinorFeatures5_UNK3 0x00000008 +#define chipMinorFeatures4_BUG_FIXES19 0x40000000 +#define chipMinorFeatures4_SH_ENHANCEMENTS6 0x80000000 +#define chipMinorFeatures5_SH_ENHANCEMENTS7 0x00000001 +#define chipMinorFeatures5_BUG_FIXES20 0x00000002 +#define chipMinorFeatures5_DE_ADDRESS_40 0x00000004 +#define chipMinorFeatures5_MINI_MMU_FIX 0x00000008 #define chipMinorFeatures5_EEZ 0x00000010 -#define chipMinorFeatures5_UNK5 0x00000020 -#define chipMinorFeatures5_UNK6 0x00000040 -#define chipMinorFeatures5_UNK7 0x00000080 -#define chipMinorFeatures5_UNK8 0x00000100 +#define chipMinorFeatures5_BUG_FIXES21 0x00000020 +#define chipMinorFeatures5_EXTRA_VG_CAPS 0x00000040 +#define chipMinorFeatures5_MULTI_SRC_V15 0x00000080 +#define chipMinorFeatures5_BUG_FIXES22 0x00000100 #define chipMinorFeatures5_HALTI3 0x00000200 -#define chipMinorFeatures5_UNK10 0x00000400 +#define chipMinorFeatures5_TESSELATION_SHADERS 0x00000400 #define chipMinorFeatures5_2D_ONE_PASS_FILTER_TAP 0x00000800 -#define chipMinorFeatures5_UNK12 0x00001000 +#define chipMinorFeatures5_MULTI_SRC_V2_STR_QUAD 0x00001000 #define chipMinorFeatures5_SEPARATE_SRC_DST 0x00002000 #define chipMinorFeatures5_HALTI4 0x00004000 -#define chipMinorFeatures5_UNK15 0x00008000 +#define chipMinorFeatures5_RA_WRITE_DEPTH 0x00008000 #define chipMinorFeatures5_ANDROID_ONLY 0x00010000 #define chipMinorFeatures5_HAS_PRODUCTID 0x00020000 -#define chipMinorFeatures5_UNK18 0x00040000 -#define chipMinorFeatures5_UNK19 0x00080000 +#define chipMinorFeatures5_TX_SUPPORT_DEC 0x00040000 +#define chipMinorFeatures5_S8_MSAA_COMPRESSION 0x00080000 #define chipMinorFeatures5_PE_DITHER_FIX2 0x00100000 -#define chipMinorFeatures5_UNK21 0x00200000 -#define chipMinorFeatures5_UNK22 0x00400000 -#define chipMinorFeatures5_UNK23 0x00800000 -#define chipMinorFeatures5_UNK24 0x01000000 -#define chipMinorFeatures5_UNK25 0x02000000 -#define chipMinorFeatures5_UNK26 0x04000000 +#define chipMinorFeatures5_L2_CACHE_REMOVE 0x00200000 +#define chipMinorFeatures5_FE_ALLOW_RND_VTX_CNT 0x00400000 +#define chipMinorFeatures5_CUBE_MAP_FL28 0x00800000 +#define chipMinorFeatures5_TX_6BIT_FRAC 0x01000000 +#define chipMinorFeatures5_FE_ALLOW_STALL_PREFETCH_ENG 0x02000000 +#define chipMinorFeatures5_THIRD_PARTY_COMPRESSION 0x04000000 #define chipMinorFeatures5_RS_DEPTHSTENCIL_NATIVE_SUPPORT 0x08000000 #define chipMinorFeatures5_V2_MSAA_COMP_FIX 0x10000000 -#define chipMinorFeatures5_UNK29 0x20000000 -#define chipMinorFeatures5_UNK30 0x40000000 -#define chipMinorFeatures5_UNK31 0x80000000 +#define chipMinorFeatures5_HALTI5 0x20000000 +#define chipMinorFeatures5_EVIS 0x40000000 +#define chipMinorFeatures5_BLT_ENGINE 0x80000000 +#define chipMinorFeatures6_BUG_FIXES_23 0x00000001 +#define chipMinorFeatures6_BUG_FIXES_24 0x00000002 +#define chipMinorFeatures6_DEC 0x00000004 +#define chipMinorFeatures6_VS_TILE_NV12 0x00000008 +#define chipMinorFeatures6_VS_TILE_NV12_10BIT 0x00000010 +#define chipMinorFeatures6_RENDER_TARGET_8 0x00000020 +#define chipMinorFeatures6_TEX_LOD_FLOW_CORR 0x00000040 +#define chipMinorFeatures6_FACE_LOD 0x00000080 +#define chipMinorFeatures6_MULTI_CORE_SEMAPHORE_STALL_V2 0x00000100 +#define chipMinorFeatures6_VMSAA 0x00000200 +#define chipMinorFeatures6_CHIP_ENABLE_LINK 0x00000400 +#define chipMinorFeatures6_MULTI_SRC_BLT_1_5_ENHANCEMENT 0x00000800 +#define chipMinorFeatures6_MULTI_SRC_BLT_BILINEAR_FILTER 0x00001000 +#define chipMinorFeatures6_RA_HZEZ_CLOCK_CONTROL 0x00002000 +#define chipMinorFeatures6_CACHE128B256BPERLINE 0x00004000 +#define chipMinorFeatures6_V4_COMPRESSION 0x00008000 +#define chipMinorFeatures6_PE2D_MAJOR_SUPER_TILE 0x00010000 +#define chipMinorFeatures6_PE_32BPC_COLORMASK_FIX 0x00020000 +#define chipMinorFeatures6_ALPHA_BLENDING_OPT 0x00040000 +#define chipMinorFeatures6_NEW_GPIPE 0x00080000 +#define chipMinorFeatures6_PIPELINE_32_ATTRIBUTES 0x00100000 +#define chipMinorFeatures6_MSAA_SHADING 0x00200000 +#define chipMinorFeatures6_NO_ANISTRO_FILTER 0x00400000 +#define chipMinorFeatures6_NO_ASTC 0x00800000 +#define chipMinorFeatures6_NO_DXT 0x01000000 +#define chipMinorFeatures6_HWTFB 0x02000000 +#define chipMinorFeatures6_RA_DEPTH_WRITE_MSAA1X_FIX 0x04000000 +#define chipMinorFeatures6_EZHZ_CLOCKGATE_FIX 0x08000000 +#define chipMinorFeatures6_SH_SNAP2PAGE_FIX 0x10000000 +#define chipMinorFeatures6_SH_HALFDEPENDENCY_FIX 0x20000000 +#define chipMinorFeatures6_USC_MCFILL_FIX 0x40000000 +#define chipMinorFeatures6_TPG_TCPERF_FIX 0x80000000 +#define chipMinorFeatures7_USC_MDFIFO_OVERFLOW_FIX 0x00000001 +#define chipMinorFeatures7_SH_TEXLD_BARRIER_IN_CS_FIX 0x00000002 +#define chipMinorFeatures7_RS_NEW_BASEADDR 0x00000004 +#define chipMinorFeatures7_PE_8BPP_DUALPIPE_FIX 0x00000008 +#define chipMinorFeatures7_SH_ADVANCED_INSTR 0x00000010 +#define chipMinorFeatures7_SH_FLAT_INTERPOLATION_DUAL16_FIX 0x00000020 +#define chipMinorFeatures7_USC_CONTINUOUS_FLUS_FIX 0x00000040 +#define chipMinorFeatures7_SH_SUPPORT_V4 0x00000080 +#define chipMinorFeatures7_SH_SUPPORT_ALPHA_KILL 0x00000100 +#define chipMinorFeatures7_PE_NO_ALPHA_TEST 0x00000200 +#define chipMinorFeatures7_TX_LOD_NEAREST_SELECT 0x00000400 +#define chipMinorFeatures7_SH_FIX_LDEXP 0x00000800 +#define chipMinorFeatures7_SUPPORT_MOVAI 0x00001000 +#define chipMinorFeatures7_SH_SNAP2PAGE_MAXPAGES_FIX 0x00002000 +#define chipMinorFeatures7_PE_RGBA16I_FIX 0x00004000 +#define chipMinorFeatures7_BLT_8bpp_256TILE_FC_FIX 0x00008000 +#define chipMinorFeatures7_PE_64BIT_FENCE_FIX 0x00010000 +#define chipMinorFeatures7_USC_FULL_CACHE_FIX 0x00020000 +#define chipMinorFeatures7_TX_YUV_ASSEMBLER_10BIT 0x00040000 +#define chipMinorFeatures7_FE_32BIT_INDEX_FIX 0x00080000 +#define chipMinorFeatures7_BLT_64BPP_MASKED_CLEAR_FIX 0x00100000 +#define chipMinorFeatures7_BIT_SECURITY 0x00200000 +#define chipMinorFeatures7_BIT_ROBUSTNESS 0x00400000 +#define chipMinorFeatures7_USC_ATOMIC_FIX 0x00800000 +#define chipMinorFeatures7_SH_PSO_MSAA1x_FIX 0x01000000 +#define chipMinorFeatures7_BIT_USC_VX_PERF_FIX 0x02000000 +#define chipMinorFeatures7_EVIS_NO_ABSDIFF 0x04000000 +#define chipMinorFeatures7_EVIS_NO_BITREPLACE 0x08000000 +#define chipMinorFeatures7_EVIS_NO_BOXFILTER 0x10000000 +#define chipMinorFeatures7_EVIS_NO_CORDIAC 0x20000000 +#define chipMinorFeatures7_EVIS_NO_DP32 0x40000000 +#define chipMinorFeatures7_EVIS_NO_FILTER 0x80000000 +#define chipMinorFeatures8_EVIS_NO_IADD 0x00000001 +#define chipMinorFeatures8_EVIS_NO_SELECTADD 0x00000002 +#define chipMinorFeatures8_EVIS_LERP_7OUTPUT 0x00000004 +#define chipMinorFeatures8_EVIS_ACCSQ_8OUTPUT 0x00000008 +#define chipMinorFeatures8_USC_GOS_ADDR_FIX 0x00000010 +#define chipMinorFeatures8_TX_8BIT_UVFRAC 0x00000020 +#define chipMinorFeatures8_TX_DESC_CACHE_CLOCKGATE_FIX 0x00000040 +#define chipMinorFeatures8_RSBLT_MSAA_DECOMPRESSION 0x00000080 +#define chipMinorFeatures8_TX_INTEGER_COORDINATE 0x00000100 +#define chipMinorFeatures8_DRAWID 0x00000200 +#define chipMinorFeatures8_PSIO_SAMPLEMASK_IN_R0ZW_FIX 0x00000400 +#define chipMinorFeatures8_TX_INTEGER_COORDINATE_V2 0x00000800 +#define chipMinorFeatures8_MULTI_CORE_BLOCK_SET_CONFIG 0x00001000 +#define chipMinorFeatures8_VG_RESOLVE_ENGINE 0x00002000 +#define chipMinorFeatures8_VG_PE_COLOR_KEY 0x00004000 +#define chipMinorFeatures8_VG_IM_INDEX_FORMAT 0x00008000 +#define chipMinorFeatures8_SNAPPAGE_CMD 0x00010000 +#define chipMinorFeatures8_SH_NO_INDEX_CONST_ON_A0 0x00020000 +#define chipMinorFeatures8_SH_NO_ONECONST_LIMIT 0x00040000 +#define chipMinorFeatures8_SH_IMG_LDST_ON_TEMP 0x00080000 +#define chipMinorFeatures8_COMPUTE_ONLY 0x00100000 +#define chipMinorFeatures8_SH_IMG_LDST_CLAMP 0x00200000 +#define chipMinorFeatures8_SH_ICACHE_ALLOC_COUNT_FIX 0x00400000 +#define chipMinorFeatures8_SH_ICACHE_PREFETCH 0x00800000 +#define chipMinorFeatures8_PE2D_SEPARATE_CACHE 0x01000000 +#define chipMinorFeatures8_VG_AYUV_INPUT_OUTPUT 0x02000000 +#define chipMinorFeatures8_VG_DOUBLE_IMAGE 0x04000000 +#define chipMinorFeatures8_VG_RECTANGLE_STRIPE_MODE 0x08000000 +#define chipMinorFeatures8_VG_MMU 0x10000000 +#define chipMinorFeatures8_VG_IM_FILTER 0x20000000 +#define chipMinorFeatures8_VG_IM_YUV_PACKET 0x40000000 +#define chipMinorFeatures8_VG_IM_YUV_PLANAR 0x80000000 +#define chipMinorFeatures9_VG_PE_YUV_PACKET 0x00000001 +#define chipMinorFeatures9_VG_COLOR_PRECISION_8_BIT 0x00000002 +#define chipMinorFeatures9_PE_MSAA_OQ_FIX 0x00000004 +#define chipMinorFeatures9_PSIO_MSAA_CL_FIX 0x00000008 +#define chipMinorFeatures9_USC_DEFER_FILL_FIX 0x00000010 +#define chipMinorFeatures9_SH_CLOCK_GATE_FIX 0x00000020 +#define chipMinorFeatures9_FE_NEED_DUMMYDRAW 0x00000040 +#define chipMinorFeatures9_PE2D_LINEAR_YUV420_OUTPUT 0x00000080 +#define chipMinorFeatures9_PE2D_LINEAR_YUV420_10BIT 0x00000100 +#define chipMinorFeatures9_MULTI_CLUSTER 0x00000200 +#define chipMinorFeatures9_VG_TS_CULLING 0x00000400 +#define chipMinorFeatures9_VG_FP25 0x00000800 +#define chipMinorFeatures9_SH_MULTI_WG_PACK 0x00001000 +#define chipMinorFeatures9_SH_DUAL16_SAMPLEMASK_ZW 0x00002000 +#define chipMinorFeatures9_TPG_TRIVIAL_MODE_FIX 0x00004000 +#define chipMinorFeatures9_TX_ASTC_MULTISLICE_FIX 0x00008000 +#define chipMinorFeatures9_FE_ROBUST_FIX 0x00010000 +#define chipMinorFeatures9_SH_GPIPE_ACCESS_FULLTEMPS 0x00020000 +#define chipMinorFeatures9_PSIO_INTERLOCK 0x00040000 +#define chipMinorFeatures9_PA_WIDELINE_FIX 0x00080000 +#define chipMinorFeatures9_WIDELINE_HELPER_FIX 0x00100000 +#define chipMinorFeatures9_G2D_3RD_PARTY_COMPRESSION_1_1 0x00200000 +#define chipMinorFeatures9_TX_FLUSH_L1CACHE 0x00400000 +#define chipMinorFeatures9_PE_DITHER_FIX2 0x00800000 +#define chipMinorFeatures9_G2D_DEC400 0x01000000 +#define chipMinorFeatures9_SH_TEXLD_U_FIX 0x02000000 +#define chipMinorFeatures9_MC_FCCACHE_BYTEMASK 0x04000000 +#define chipMinorFeatures9_SH_MULTI_WG_PACK_FIX 0x08000000 +#define chipMinorFeatures9_DC_OVERLAY_SCALING 0x10000000 +#define chipMinorFeatures9_DC_SOURCE_ROTATION 0x20000000 +#define chipMinorFeatures9_DC_TILED 0x40000000 +#define chipMinorFeatures9_DC_YUV_L1 0x80000000 +#define chipMinorFeatures10_DC_D30_OUTPUT 0x00000001 +#define chipMinorFeatures10_DC_MMU 0x00000002 +#define chipMinorFeatures10_DC_COMPRESSION 0x00000004 +#define chipMinorFeatures10_DC_QOS 0x00000008 +#define chipMinorFeatures10_PE_ADVANCE_BLEND_PART0 0x00000010 +#define chipMinorFeatures10_FE_PATCHLIST_FETCH_FIX 0x00000020 +#define chipMinorFeatures10_RA_CG_FIX 0x00000040 +#define chipMinorFeatures10_EVIS_VX2 0x00000080 +#define chipMinorFeatures10_NN_FLOAT 0x00000100 +#define chipMinorFeatures10_DEC400 0x00000200 +#define chipMinorFeatures10_LS_SUPPORT_PERCOMP_DEPENDENCY 0x00000400 +#define chipMinorFeatures10_TP_ENGINE 0x00000800 +#define chipMinorFeatures10_MULTI_CORE_BLOCK_SET_CONFIG2 0x00001000 +#define chipMinorFeatures10_PE_VMSAA_COVERAGE_CACHE_FIX 0x00002000 +#define chipMinorFeatures10_SECURITY_AHB 0x00004000 +#define chipMinorFeatures10_MULTICORE_SEMAPHORESTALL_V3 0x00008000 +#define chipMinorFeatures10_SMALLBATCH 0x00010000 +#define chipMinorFeatures10_SH_CMPLX 0x00020000 +#define chipMinorFeatures10_SH_IDIV0_SWZL_EHS 0x00040000 +#define chipMinorFeatures10_TX_LERP_LESS_BIT 0x00080000 +#define chipMinorFeatures10_SH_GM_ENDIAN 0x00100000 +#define chipMinorFeatures10_SH_GM_USC_UNALLOC 0x00200000 +#define chipMinorFeatures10_SH_END_OF_BB 0x00400000 +#define chipMinorFeatures10_VIP_V7 0x00800000 +#define chipMinorFeatures10_TX_BORDER_CLAMP_FIX 0x01000000 +#define chipMinorFeatures10_SH_IMG_LD_LASTPIXEL_FIX 0x02000000 +#define chipMinorFeatures10_ASYNC_BLT 0x04000000 +#define chipMinorFeatures10_ASYNC_FE_FENCE_FIX 0x08000000 +#define chipMinorFeatures10_PSCS_THROTTLE 0x10000000 +#define chipMinorFeatures10_SEPARATE_LS 0x20000000 +#define chipMinorFeatures10_MCFE 0x40000000 +#define chipMinorFeatures10_WIDELINE_TRIANGLE_EMU 0x80000000 +#define chipMinorFeatures11_VG_RESOLUTION_8K 0x00000001 +#define chipMinorFeatures11_FENCE_32BIT 0x00000002 +#define chipMinorFeatures11_FENCE_64BIT 0x00000004 +#define chipMinorFeatures11_NN_INTERLEVE8 0x00000008 +#define chipMinorFeatures11_TP_REORDER 0x00000010 +#define chipMinorFeatures11_PE_DEPTH_ONLY_OQFIX 0x00000020 #endif /* COMMON_XML */ diff --git a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c index 99ad2f073c6e..bfc6d4aa3b7c 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_buffer.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_buffer.c @@ -215,6 +215,24 @@ u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe return buffer->user_size / 8; } +u16 etnaviv_buffer_config_pta(struct etnaviv_gpu *gpu) +{ + struct etnaviv_cmdbuf *buffer = &gpu->buffer; + + lockdep_assert_held(&gpu->lock); + + buffer->user_size = 0; + + CMD_LOAD_STATE(buffer, VIVS_MMUv2_PTA_CONFIG, + VIVS_MMUv2_PTA_CONFIG_INDEX(0)); + + CMD_END(buffer); + + buffer->user_size = ALIGN(buffer->user_size, 8); + + return buffer->user_size / 8; +} + void etnaviv_buffer_end(struct etnaviv_gpu *gpu) { struct etnaviv_cmdbuf *buffer = &gpu->buffer; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.c b/drivers/gpu/drm/etnaviv/etnaviv_drv.c index 6faf4042db23..ab50090d066c 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.c @@ -101,12 +101,25 @@ static void load_gpu(struct drm_device *dev) static int etnaviv_open(struct drm_device *dev, struct drm_file *file) { + struct etnaviv_drm_private *priv = dev->dev_private; struct etnaviv_file_private *ctx; + int i; ctx = kzalloc(sizeof(*ctx), GFP_KERNEL); if (!ctx) return -ENOMEM; + for (i = 0; i < ETNA_MAX_PIPES; i++) { + struct etnaviv_gpu *gpu = priv->gpu[i]; + + if (gpu) { + drm_sched_entity_init(&gpu->sched, + &ctx->sched_entity[i], + &gpu->sched.sched_rq[DRM_SCHED_PRIORITY_NORMAL], + 32, NULL); + } + } + file->driver_priv = ctx; return 0; @@ -126,6 +139,9 @@ static void etnaviv_postclose(struct drm_device *dev, struct drm_file *file) if (gpu->lastctx == ctx) gpu->lastctx = NULL; mutex_unlock(&gpu->lock); + + drm_sched_entity_fini(&gpu->sched, + &ctx->sched_entity[i]); } } @@ -637,25 +653,21 @@ static int compare_str(struct device *dev, void *data) static int etnaviv_pdev_probe(struct platform_device *pdev) { struct device *dev = &pdev->dev; - struct device_node *node = dev->of_node; struct component_match *match = NULL; dma_set_coherent_mask(&pdev->dev, DMA_BIT_MASK(32)); - if (node) { + if (!dev->platform_data) { struct device_node *core_node; - int i; - for (i = 0; ; i++) { - core_node = of_parse_phandle(node, "cores", i); - if (!core_node) - break; + for_each_compatible_node(core_node, NULL, "vivante,gc") { + if (!of_device_is_available(core_node)) + continue; drm_of_component_match_add(&pdev->dev, &match, compare_of, core_node); - of_node_put(core_node); } - } else if (dev->platform_data) { + } else { char **names = dev->platform_data; unsigned i; @@ -673,25 +685,18 @@ static int etnaviv_pdev_remove(struct platform_device *pdev) return 0; } -static const struct of_device_id dt_match[] = { - { .compatible = "fsl,imx-gpu-subsystem" }, - { .compatible = "marvell,dove-gpu-subsystem" }, - {} -}; -MODULE_DEVICE_TABLE(of, dt_match); - static struct platform_driver etnaviv_platform_driver = { .probe = etnaviv_pdev_probe, .remove = etnaviv_pdev_remove, .driver = { .name = "etnaviv", - .of_match_table = dt_match, }, }; static int __init etnaviv_init(void) { int ret; + struct device_node *np; etnaviv_validate_init(); @@ -703,6 +708,19 @@ static int __init etnaviv_init(void) if (ret != 0) platform_driver_unregister(&etnaviv_gpu_driver); + /* + * If the DT contains at least one available GPU device, instantiate + * the DRM platform device. + */ + for_each_compatible_node(np, NULL, "vivante,gc") { + if (!of_device_is_available(np)) + continue; + + platform_device_register_simple("etnaviv", -1, NULL, 0); + of_node_put(np); + break; + } + return ret; } module_init(etnaviv_init); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_drv.h b/drivers/gpu/drm/etnaviv/etnaviv_drv.h index a54f0b758a5c..ddb17ee565e9 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_drv.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_drv.h @@ -34,6 +34,7 @@ #include <drm/drm_fb_helper.h> #include <drm/drm_gem.h> #include <drm/etnaviv_drm.h> +#include <drm/gpu_scheduler.h> struct etnaviv_cmdbuf; struct etnaviv_gpu; @@ -42,11 +43,11 @@ struct etnaviv_gem_object; struct etnaviv_gem_submit; struct etnaviv_file_private { - /* currently we don't do anything useful with this.. but when - * per-context address spaces are supported we'd keep track of + /* + * When per-context address spaces are supported we'd keep track of * the context's page-tables here. */ - int dummy; + struct drm_sched_entity sched_entity[ETNA_MAX_PIPES]; }; struct etnaviv_drm_private { @@ -85,6 +86,7 @@ int etnaviv_gem_new_userptr(struct drm_device *dev, struct drm_file *file, uintptr_t ptr, u32 size, u32 flags, u32 *handle); u16 etnaviv_buffer_init(struct etnaviv_gpu *gpu); u16 etnaviv_buffer_config_mmuv2(struct etnaviv_gpu *gpu, u32 mtlb_addr, u32 safe_addr); +u16 etnaviv_buffer_config_pta(struct etnaviv_gpu *gpu); void etnaviv_buffer_end(struct etnaviv_gpu *gpu); void etnaviv_sync_point_queue(struct etnaviv_gpu *gpu, unsigned int event); void etnaviv_buffer_queue(struct etnaviv_gpu *gpu, u32 exec_state, diff --git a/drivers/gpu/drm/etnaviv/etnaviv_dump.c b/drivers/gpu/drm/etnaviv/etnaviv_dump.c index 6d0909c589d1..48aef6cf6a42 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_dump.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_dump.c @@ -20,9 +20,13 @@ #include "etnaviv_gem.h" #include "etnaviv_gpu.h" #include "etnaviv_mmu.h" +#include "etnaviv_sched.h" #include "state.xml.h" #include "state_hi.xml.h" +static bool etnaviv_dump_core = true; +module_param_named(dump_core, etnaviv_dump_core, bool, 0600); + struct core_dump_iterator { void *start; struct etnaviv_dump_object_header *hdr; @@ -121,10 +125,16 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu) struct etnaviv_vram_mapping *vram; struct etnaviv_gem_object *obj; struct etnaviv_gem_submit *submit; + struct drm_sched_job *s_job; unsigned int n_obj, n_bomap_pages; size_t file_size, mmu_size; __le64 *bomap, *bomap_start; + /* Only catch the first event, or when manually re-armed */ + if (!etnaviv_dump_core) + return; + etnaviv_dump_core = false; + mmu_size = etnaviv_iommu_dump_size(gpu->mmu); /* We always dump registers, mmu, ring and end marker */ @@ -135,10 +145,13 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu) mmu_size + gpu->buffer.size; /* Add in the active command buffers */ - list_for_each_entry(submit, &gpu->active_submit_list, node) { + spin_lock(&gpu->sched.job_list_lock); + list_for_each_entry(s_job, &gpu->sched.ring_mirror_list, node) { + submit = to_etnaviv_submit(s_job); file_size += submit->cmdbuf.size; n_obj++; } + spin_unlock(&gpu->sched.job_list_lock); /* Add in the active buffer objects */ list_for_each_entry(vram, &gpu->mmu->mappings, mmu_node) { @@ -180,10 +193,14 @@ void etnaviv_core_dump(struct etnaviv_gpu *gpu) gpu->buffer.size, etnaviv_cmdbuf_get_va(&gpu->buffer)); - list_for_each_entry(submit, &gpu->active_submit_list, node) + spin_lock(&gpu->sched.job_list_lock); + list_for_each_entry(s_job, &gpu->sched.ring_mirror_list, node) { + submit = to_etnaviv_submit(s_job); etnaviv_core_dump_mem(&iter, ETDUMP_BUF_CMD, submit->cmdbuf.vaddr, submit->cmdbuf.size, etnaviv_cmdbuf_get_va(&submit->cmdbuf)); + } + spin_unlock(&gpu->sched.job_list_lock); /* Reserve space for the bomap */ if (n_bomap_pages) { diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem.h b/drivers/gpu/drm/etnaviv/etnaviv_gem.h index be72a9833f2b..93e696fcc14f 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem.h @@ -94,6 +94,9 @@ struct etnaviv_gem_submit_bo { u32 flags; struct etnaviv_gem_object *obj; struct etnaviv_vram_mapping *mapping; + struct dma_fence *excl; + unsigned int nr_shared; + struct dma_fence **shared; }; /* Created per submit-ioctl, to track bo's and cmdstream bufs, etc, @@ -101,9 +104,11 @@ struct etnaviv_gem_submit_bo { * make it easier to unwind when things go wrong, etc). */ struct etnaviv_gem_submit { + struct drm_sched_job sched_job; struct kref refcount; struct etnaviv_gpu *gpu; struct dma_fence *out_fence, *in_fence; + int out_fence_id; struct list_head node; /* GPU active submit list */ struct etnaviv_cmdbuf cmdbuf; bool runtime_resumed; diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c index 1f8202bca061..46ecd3e66ac9 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gem_submit.c @@ -22,6 +22,7 @@ #include "etnaviv_gpu.h" #include "etnaviv_gem.h" #include "etnaviv_perfmon.h" +#include "etnaviv_sched.h" /* * Cmdstream submission: @@ -169,29 +170,33 @@ fail: return ret; } -static int submit_fence_sync(const struct etnaviv_gem_submit *submit) +static int submit_fence_sync(struct etnaviv_gem_submit *submit) { - unsigned int context = submit->gpu->fence_context; int i, ret = 0; for (i = 0; i < submit->nr_bos; i++) { - struct etnaviv_gem_object *etnaviv_obj = submit->bos[i].obj; - bool write = submit->bos[i].flags & ETNA_SUBMIT_BO_WRITE; - bool explicit = !!(submit->flags & ETNA_SUBMIT_NO_IMPLICIT); + struct etnaviv_gem_submit_bo *bo = &submit->bos[i]; + struct reservation_object *robj = bo->obj->resv; - ret = etnaviv_gpu_fence_sync_obj(etnaviv_obj, context, write, - explicit); - if (ret) - break; - } + if (!(bo->flags & ETNA_SUBMIT_BO_WRITE)) { + ret = reservation_object_reserve_shared(robj); + if (ret) + return ret; + } + + if (submit->flags & ETNA_SUBMIT_NO_IMPLICIT) + continue; + + if (bo->flags & ETNA_SUBMIT_BO_WRITE) { + ret = reservation_object_get_fences_rcu(robj, &bo->excl, + &bo->nr_shared, + &bo->shared); + if (ret) + return ret; + } else { + bo->excl = reservation_object_get_excl_rcu(robj); + } - if (submit->flags & ETNA_SUBMIT_FENCE_FD_IN) { - /* - * Wait if the fence is from a foreign context, or if the fence - * array contains any fence from a foreign context. - */ - if (!dma_fence_match_context(submit->in_fence, context)) - ret = dma_fence_wait(submit->in_fence, true); } return ret; @@ -381,8 +386,13 @@ static void submit_cleanup(struct kref *kref) if (submit->in_fence) dma_fence_put(submit->in_fence); - if (submit->out_fence) + if (submit->out_fence) { + /* first remove from IDR, so fence can not be found anymore */ + mutex_lock(&submit->gpu->fence_idr_lock); + idr_remove(&submit->gpu->fence_idr, submit->out_fence_id); + mutex_unlock(&submit->gpu->fence_idr_lock); dma_fence_put(submit->out_fence); + } kfree(submit->pmrs); kfree(submit); } @@ -395,6 +405,7 @@ void etnaviv_submit_put(struct etnaviv_gem_submit *submit) int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, struct drm_file *file) { + struct etnaviv_file_private *ctx = file->driver_priv; struct etnaviv_drm_private *priv = dev->dev_private; struct drm_etnaviv_gem_submit *args = data; struct drm_etnaviv_gem_submit_reloc *relocs; @@ -503,10 +514,6 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, if (ret) goto err_submit_objects; - ret = submit_lock_objects(submit, &ticket); - if (ret) - goto err_submit_objects; - if (!etnaviv_cmd_validate_one(gpu, stream, args->stream_size / 4, relocs, args->nr_relocs)) { ret = -EINVAL; @@ -521,10 +528,6 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, } } - ret = submit_fence_sync(submit); - if (ret) - goto err_submit_objects; - ret = submit_pin_objects(submit); if (ret) goto err_submit_objects; @@ -539,9 +542,16 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, goto err_submit_objects; memcpy(submit->cmdbuf.vaddr, stream, args->stream_size); - submit->cmdbuf.user_size = ALIGN(args->stream_size, 8); - ret = etnaviv_gpu_submit(gpu, submit); + ret = submit_lock_objects(submit, &ticket); + if (ret) + goto err_submit_objects; + + ret = submit_fence_sync(submit); + if (ret) + goto err_submit_objects; + + ret = etnaviv_sched_push_job(&ctx->sched_entity[args->pipe], submit); if (ret) goto err_submit_objects; @@ -563,7 +573,7 @@ int etnaviv_ioctl_gem_submit(struct drm_device *dev, void *data, } args->fence_fd = out_fence_fd; - args->fence = submit->out_fence->seqno; + args->fence = submit->out_fence_id; err_submit_objects: etnaviv_submit_put(submit); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c index 21d0d22f1168..8a88799bf79b 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.c @@ -26,19 +26,21 @@ #include "etnaviv_gem.h" #include "etnaviv_mmu.h" #include "etnaviv_perfmon.h" +#include "etnaviv_sched.h" #include "common.xml.h" #include "state.xml.h" #include "state_hi.xml.h" #include "cmdstream.xml.h" +#ifndef PHYS_OFFSET +#define PHYS_OFFSET 0 +#endif + static const struct platform_device_id gpu_ids[] = { { .name = "etnaviv-gpu,2d" }, { }, }; -static bool etnaviv_dump_core = true; -module_param_named(dump_core, etnaviv_dump_core, bool, 0600); - /* * Driver functions: */ @@ -82,6 +84,30 @@ int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value) *value = gpu->identity.minor_features5; break; + case ETNAVIV_PARAM_GPU_FEATURES_7: + *value = gpu->identity.minor_features6; + break; + + case ETNAVIV_PARAM_GPU_FEATURES_8: + *value = gpu->identity.minor_features7; + break; + + case ETNAVIV_PARAM_GPU_FEATURES_9: + *value = gpu->identity.minor_features8; + break; + + case ETNAVIV_PARAM_GPU_FEATURES_10: + *value = gpu->identity.minor_features9; + break; + + case ETNAVIV_PARAM_GPU_FEATURES_11: + *value = gpu->identity.minor_features10; + break; + + case ETNAVIV_PARAM_GPU_FEATURES_12: + *value = gpu->identity.minor_features11; + break; + case ETNAVIV_PARAM_GPU_STREAM_COUNT: *value = gpu->identity.stream_count; break; @@ -348,6 +374,13 @@ static void etnaviv_hw_identify(struct etnaviv_gpu *gpu) dev_info(gpu->dev, "model: GC%x, revision: %x\n", gpu->identity.model, gpu->identity.revision); + /* + * If there is a match in the HWDB, we aren't interested in the + * remaining register values, as they might be wrong. + */ + if (etnaviv_fill_identity_from_hwdb(gpu)) + return; + gpu->identity.features = gpu_read(gpu, VIVS_HI_CHIP_FEATURE); /* Disable fast clear on GC700. */ @@ -448,9 +481,14 @@ static int etnaviv_hw_reset(struct etnaviv_gpu *gpu) control |= VIVS_HI_CLOCK_CONTROL_ISOLATE_GPU; gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); - /* set soft reset. */ - control |= VIVS_HI_CLOCK_CONTROL_SOFT_RESET; - gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); + if (gpu->sec_mode == ETNA_SEC_KERNEL) { + gpu_write(gpu, VIVS_MMUv2_AHB_CONTROL, + VIVS_MMUv2_AHB_CONTROL_RESET); + } else { + /* set soft reset. */ + control |= VIVS_HI_CLOCK_CONTROL_SOFT_RESET; + gpu_write(gpu, VIVS_HI_CLOCK_CONTROL, control); + } /* wait for reset. */ usleep_range(10, 20); @@ -561,6 +599,12 @@ void etnaviv_gpu_start_fe(struct etnaviv_gpu *gpu, u32 address, u16 prefetch) gpu_write(gpu, VIVS_FE_COMMAND_CONTROL, VIVS_FE_COMMAND_CONTROL_ENABLE | VIVS_FE_COMMAND_CONTROL_PREFETCH(prefetch)); + + if (gpu->sec_mode == ETNA_SEC_KERNEL) { + gpu_write(gpu, VIVS_MMUv2_SEC_COMMAND_CONTROL, + VIVS_MMUv2_SEC_COMMAND_CONTROL_ENABLE | + VIVS_MMUv2_SEC_COMMAND_CONTROL_PREFETCH(prefetch)); + } } static void etnaviv_gpu_setup_pulse_eater(struct etnaviv_gpu *gpu) @@ -634,6 +678,12 @@ static void etnaviv_gpu_hw_init(struct etnaviv_gpu *gpu) gpu_write(gpu, VIVS_MC_BUS_CONFIG, bus_config); } + if (gpu->sec_mode == ETNA_SEC_KERNEL) { + u32 val = gpu_read(gpu, VIVS_MMUv2_AHB_CONTROL); + val |= VIVS_MMUv2_AHB_CONTROL_NONSEC_ACCESS; + gpu_write(gpu, VIVS_MMUv2_AHB_CONTROL, val); + } + /* setup the pulse eater */ etnaviv_gpu_setup_pulse_eater(gpu); @@ -696,6 +746,14 @@ int etnaviv_gpu_init(struct etnaviv_gpu *gpu) gpu->identity.features &= ~chipFeatures_FAST_CLEAR; } + /* + * On cores with security features supported, we claim control over the + * security states. + */ + if ((gpu->identity.minor_features7 & chipMinorFeatures7_BIT_SECURITY) && + (gpu->identity.minor_features10 & chipMinorFeatures10_SECURITY_AHB)) + gpu->sec_mode = ETNA_SEC_KERNEL; + ret = etnaviv_hw_reset(gpu); if (ret) { dev_err(gpu->dev, "GPU reset failed\n"); @@ -807,6 +865,8 @@ int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m) verify_dma(gpu, &debug); seq_puts(m, "\tfeatures\n"); + seq_printf(m, "\t major_features: 0x%08x\n", + gpu->identity.features); seq_printf(m, "\t minor_features0: 0x%08x\n", gpu->identity.minor_features0); seq_printf(m, "\t minor_features1: 0x%08x\n", @@ -819,6 +879,18 @@ int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m) gpu->identity.minor_features4); seq_printf(m, "\t minor_features5: 0x%08x\n", gpu->identity.minor_features5); + seq_printf(m, "\t minor_features6: 0x%08x\n", + gpu->identity.minor_features6); + seq_printf(m, "\t minor_features7: 0x%08x\n", + gpu->identity.minor_features7); + seq_printf(m, "\t minor_features8: 0x%08x\n", + gpu->identity.minor_features8); + seq_printf(m, "\t minor_features9: 0x%08x\n", + gpu->identity.minor_features9); + seq_printf(m, "\t minor_features10: 0x%08x\n", + gpu->identity.minor_features10); + seq_printf(m, "\t minor_features11: 0x%08x\n", + gpu->identity.minor_features11); seq_puts(m, "\tspecs\n"); seq_printf(m, "\t stream_count: %d\n", @@ -912,38 +984,24 @@ int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m) } #endif -/* - * Hangcheck detection for locked gpu: - */ -static void recover_worker(struct work_struct *work) +void etnaviv_gpu_recover_hang(struct etnaviv_gpu *gpu) { - struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu, - recover_work); unsigned long flags; unsigned int i = 0; - dev_err(gpu->dev, "hangcheck recover!\n"); + dev_err(gpu->dev, "recover hung GPU!\n"); if (pm_runtime_get_sync(gpu->dev) < 0) return; mutex_lock(&gpu->lock); - /* Only catch the first event, or when manually re-armed */ - if (etnaviv_dump_core) { - etnaviv_core_dump(gpu); - etnaviv_dump_core = false; - } - etnaviv_hw_reset(gpu); /* complete all events, the GPU won't do it after the reset */ spin_lock_irqsave(&gpu->event_spinlock, flags); - for_each_set_bit_from(i, gpu->event_bitmap, ETNA_NR_EVENTS) { - dma_fence_signal(gpu->event[i].fence); - gpu->event[i].fence = NULL; + for_each_set_bit_from(i, gpu->event_bitmap, ETNA_NR_EVENTS) complete(&gpu->event_free); - } bitmap_zero(gpu->event_bitmap, ETNA_NR_EVENTS); spin_unlock_irqrestore(&gpu->event_spinlock, flags); gpu->completed_fence = gpu->active_fence; @@ -955,56 +1013,6 @@ static void recover_worker(struct work_struct *work) mutex_unlock(&gpu->lock); pm_runtime_mark_last_busy(gpu->dev); pm_runtime_put_autosuspend(gpu->dev); - - /* Retire the buffer objects in a work */ - queue_work(gpu->wq, &gpu->retire_work); -} - -static void hangcheck_timer_reset(struct etnaviv_gpu *gpu) -{ - DBG("%s", dev_name(gpu->dev)); - mod_timer(&gpu->hangcheck_timer, - round_jiffies_up(jiffies + DRM_ETNAVIV_HANGCHECK_JIFFIES)); -} - -static void hangcheck_handler(struct timer_list *t) -{ - struct etnaviv_gpu *gpu = from_timer(gpu, t, hangcheck_timer); - u32 fence = gpu->completed_fence; - bool progress = false; - - if (fence != gpu->hangcheck_fence) { - gpu->hangcheck_fence = fence; - progress = true; - } - - if (!progress) { - u32 dma_addr = gpu_read(gpu, VIVS_FE_DMA_ADDRESS); - int change = dma_addr - gpu->hangcheck_dma_addr; - - if (change < 0 || change > 16) { - gpu->hangcheck_dma_addr = dma_addr; - progress = true; - } - } - - if (!progress && fence_after(gpu->active_fence, fence)) { - dev_err(gpu->dev, "hangcheck detected gpu lockup!\n"); - dev_err(gpu->dev, " completed fence: %u\n", fence); - dev_err(gpu->dev, " active fence: %u\n", - gpu->active_fence); - queue_work(gpu->wq, &gpu->recover_work); - } - - /* if still more pending work, reset the hangcheck timer: */ - if (fence_after(gpu->active_fence, gpu->hangcheck_fence)) - hangcheck_timer_reset(gpu); -} - -static void hangcheck_disable(struct etnaviv_gpu *gpu) -{ - del_timer_sync(&gpu->hangcheck_timer); - cancel_work_sync(&gpu->recover_work); } /* fence object management */ @@ -1080,54 +1088,6 @@ static struct dma_fence *etnaviv_gpu_fence_alloc(struct etnaviv_gpu *gpu) return &f->base; } -int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj, - unsigned int context, bool exclusive, bool explicit) -{ - struct reservation_object *robj = etnaviv_obj->resv; - struct reservation_object_list *fobj; - struct dma_fence *fence; - int i, ret; - - if (!exclusive) { - ret = reservation_object_reserve_shared(robj); - if (ret) - return ret; - } - - if (explicit) - return 0; - - /* - * If we have any shared fences, then the exclusive fence - * should be ignored as it will already have been signalled. - */ - fobj = reservation_object_get_list(robj); - if (!fobj || fobj->shared_count == 0) { - /* Wait on any existing exclusive fence which isn't our own */ - fence = reservation_object_get_excl(robj); - if (fence && fence->context != context) { - ret = dma_fence_wait(fence, true); - if (ret) - return ret; - } - } - - if (!exclusive || !fobj) - return 0; - - for (i = 0; i < fobj->shared_count; i++) { - fence = rcu_dereference_protected(fobj->shared[i], - reservation_object_held(robj)); - if (fence->context != context) { - ret = dma_fence_wait(fence, true); - if (ret) - return ret; - } - } - - return 0; -} - /* * event management: */ @@ -1194,67 +1154,47 @@ static void event_free(struct etnaviv_gpu *gpu, unsigned int event) /* * Cmdstream submission/retirement: */ - -static void retire_worker(struct work_struct *work) -{ - struct etnaviv_gpu *gpu = container_of(work, struct etnaviv_gpu, - retire_work); - u32 fence = gpu->completed_fence; - struct etnaviv_gem_submit *submit, *tmp; - LIST_HEAD(retire_list); - - mutex_lock(&gpu->lock); - list_for_each_entry_safe(submit, tmp, &gpu->active_submit_list, node) { - if (!dma_fence_is_signaled(submit->out_fence)) - break; - - list_move(&submit->node, &retire_list); - } - - gpu->retired_fence = fence; - - mutex_unlock(&gpu->lock); - - list_for_each_entry_safe(submit, tmp, &retire_list, node) - etnaviv_submit_put(submit); -} - int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu, - u32 fence, struct timespec *timeout) + u32 id, struct timespec *timeout) { + struct dma_fence *fence; int ret; - if (fence_after(fence, gpu->next_fence)) { - DRM_ERROR("waiting on invalid fence: %u (of %u)\n", - fence, gpu->next_fence); - return -EINVAL; - } + /* + * Look up the fence and take a reference. We might still find a fence + * whose refcount has already dropped to zero. dma_fence_get_rcu + * pretends we didn't find a fence in that case. + */ + rcu_read_lock(); + fence = idr_find(&gpu->fence_idr, id); + if (fence) + fence = dma_fence_get_rcu(fence); + rcu_read_unlock(); + + if (!fence) + return 0; if (!timeout) { /* No timeout was requested: just test for completion */ - ret = fence_completed(gpu, fence) ? 0 : -EBUSY; + ret = dma_fence_is_signaled(fence) ? 0 : -EBUSY; } else { unsigned long remaining = etnaviv_timeout_to_jiffies(timeout); - ret = wait_event_interruptible_timeout(gpu->fence_event, - fence_completed(gpu, fence), - remaining); - if (ret == 0) { - DBG("timeout waiting for fence: %u (retired: %u completed: %u)", - fence, gpu->retired_fence, - gpu->completed_fence); + ret = dma_fence_wait_timeout(fence, true, remaining); + if (ret == 0) ret = -ETIMEDOUT; - } else if (ret != -ERESTARTSYS) { + else if (ret != -ERESTARTSYS) ret = 0; - } + } + dma_fence_put(fence); return ret; } /* * Wait for an object to become inactive. This, on it's own, is not race - * free: the object is moved by the retire worker off the active list, and + * free: the object is moved by the scheduler off the active list, and * then the iova is put. Moreover, the object could be re-submitted just * after we notice that it's become inactive. * @@ -1343,16 +1283,19 @@ static void sync_point_perfmon_sample_post(struct etnaviv_gpu *gpu, /* add bo's to gpu's ring, and kick gpu: */ -int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, - struct etnaviv_gem_submit *submit) +struct dma_fence *etnaviv_gpu_submit(struct etnaviv_gem_submit *submit) { + struct etnaviv_gpu *gpu = submit->gpu; + struct dma_fence *gpu_fence; unsigned int i, nr_events = 1, event[3]; int ret; - ret = pm_runtime_get_sync(gpu->dev); - if (ret < 0) - return ret; - submit->runtime_resumed = true; + if (!submit->runtime_resumed) { + ret = pm_runtime_get_sync(gpu->dev); + if (ret < 0) + return NULL; + submit->runtime_resumed = true; + } /* * if there are performance monitor requests we need to have @@ -1367,21 +1310,20 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, ret = event_alloc(gpu, nr_events, event); if (ret) { DRM_ERROR("no free events\n"); - return ret; + return NULL; } mutex_lock(&gpu->lock); - submit->out_fence = etnaviv_gpu_fence_alloc(gpu); - if (!submit->out_fence) { + gpu_fence = etnaviv_gpu_fence_alloc(gpu); + if (!gpu_fence) { for (i = 0; i < nr_events; i++) event_free(gpu, event[i]); - ret = -ENOMEM; goto out_unlock; } - gpu->active_fence = submit->out_fence->seqno; + gpu->active_fence = gpu_fence->seqno; if (submit->nr_pmrs) { gpu->event[event[1]].sync_point = &sync_point_perfmon_sample_pre; @@ -1390,8 +1332,8 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, etnaviv_sync_point_queue(gpu, event[1]); } - kref_get(&submit->refcount); - gpu->event[event[0]].fence = submit->out_fence; + gpu->event[event[0]].fence = gpu_fence; + submit->cmdbuf.user_size = submit->cmdbuf.size - 8; etnaviv_buffer_queue(gpu, submit->exec_state, event[0], &submit->cmdbuf); @@ -1402,15 +1344,10 @@ int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, etnaviv_sync_point_queue(gpu, event[2]); } - list_add_tail(&submit->node, &gpu->active_submit_list); - - hangcheck_timer_reset(gpu); - ret = 0; - out_unlock: mutex_unlock(&gpu->lock); - return ret; + return gpu_fence; } static void sync_point_worker(struct work_struct *work) @@ -1428,9 +1365,35 @@ static void sync_point_worker(struct work_struct *work) etnaviv_gpu_start_fe(gpu, addr + 2, 2); } -/* - * Init/Cleanup: - */ +static void dump_mmu_fault(struct etnaviv_gpu *gpu) +{ + u32 status_reg, status; + int i; + + if (gpu->sec_mode == ETNA_SEC_NONE) + status_reg = VIVS_MMUv2_STATUS; + else + status_reg = VIVS_MMUv2_SEC_STATUS; + + status = gpu_read(gpu, status_reg); + dev_err_ratelimited(gpu->dev, "MMU fault status 0x%08x\n", status); + + for (i = 0; i < 4; i++) { + u32 address_reg; + + if (!(status & (VIVS_MMUv2_STATUS_EXCEPTION0__MASK << (i * 4)))) + continue; + + if (gpu->sec_mode == ETNA_SEC_NONE) + address_reg = VIVS_MMUv2_EXCEPTION_ADDR(i); + else + address_reg = VIVS_MMUv2_SEC_EXCEPTION_ADDR; + + dev_err_ratelimited(gpu->dev, "MMU %d fault addr 0x%08x\n", i, + gpu_read(gpu, address_reg)); + } +} + static irqreturn_t irq_handler(int irq, void *data) { struct etnaviv_gpu *gpu = data; @@ -1451,17 +1414,7 @@ static irqreturn_t irq_handler(int irq, void *data) } if (intr & VIVS_HI_INTR_ACKNOWLEDGE_MMU_EXCEPTION) { - int i; - - dev_err_ratelimited(gpu->dev, - "MMU fault status 0x%08x\n", - gpu_read(gpu, VIVS_MMUv2_STATUS)); - for (i = 0; i < 4; i++) { - dev_err_ratelimited(gpu->dev, - "MMU %d fault addr 0x%08x\n", - i, gpu_read(gpu, - VIVS_MMUv2_EXCEPTION_ADDR(i))); - } + dump_mmu_fault(gpu); intr &= ~VIVS_HI_INTR_ACKNOWLEDGE_MMU_EXCEPTION; } @@ -1484,7 +1437,6 @@ static irqreturn_t irq_handler(int irq, void *data) continue; gpu->event[event].fence = NULL; - dma_fence_signal(fence); /* * Events can be processed out of order. Eg, @@ -1497,13 +1449,11 @@ static irqreturn_t irq_handler(int irq, void *data) */ if (fence_after(fence->seqno, gpu->completed_fence)) gpu->completed_fence = fence->seqno; + dma_fence_signal(fence); event_free(gpu, event); } - /* Retire the buffer objects in a work */ - queue_work(gpu->wq, &gpu->retire_work); - ret = IRQ_HANDLED; } @@ -1514,6 +1464,12 @@ static int etnaviv_gpu_clk_enable(struct etnaviv_gpu *gpu) { int ret; + if (gpu->clk_reg) { + ret = clk_prepare_enable(gpu->clk_reg); + if (ret) + return ret; + } + if (gpu->clk_bus) { ret = clk_prepare_enable(gpu->clk_bus); if (ret) @@ -1552,6 +1508,8 @@ static int etnaviv_gpu_clk_disable(struct etnaviv_gpu *gpu) clk_disable_unprepare(gpu->clk_core); if (gpu->clk_bus) clk_disable_unprepare(gpu->clk_bus); + if (gpu->clk_reg) + clk_disable_unprepare(gpu->clk_reg); return 0; } @@ -1675,41 +1633,49 @@ static int etnaviv_gpu_bind(struct device *dev, struct device *master, gpu->wq = alloc_ordered_workqueue(dev_name(dev), 0); if (!gpu->wq) { - if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL)) - thermal_cooling_device_unregister(gpu->cooling); - return -ENOMEM; + ret = -ENOMEM; + goto out_thermal; } + ret = etnaviv_sched_init(gpu); + if (ret) + goto out_workqueue; + #ifdef CONFIG_PM ret = pm_runtime_get_sync(gpu->dev); #else ret = etnaviv_gpu_clk_enable(gpu); #endif - if (ret < 0) { - destroy_workqueue(gpu->wq); - if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL)) - thermal_cooling_device_unregister(gpu->cooling); - return ret; - } + if (ret < 0) + goto out_sched; + gpu->drm = drm; gpu->fence_context = dma_fence_context_alloc(1); + idr_init(&gpu->fence_idr); spin_lock_init(&gpu->fence_spinlock); - INIT_LIST_HEAD(&gpu->active_submit_list); - INIT_WORK(&gpu->retire_work, retire_worker); INIT_WORK(&gpu->sync_point_work, sync_point_worker); - INIT_WORK(&gpu->recover_work, recover_worker); init_waitqueue_head(&gpu->fence_event); - timer_setup(&gpu->hangcheck_timer, hangcheck_handler, TIMER_DEFERRABLE); - priv->gpu[priv->num_gpus++] = gpu; pm_runtime_mark_last_busy(gpu->dev); pm_runtime_put_autosuspend(gpu->dev); return 0; + +out_sched: + etnaviv_sched_fini(gpu); + +out_workqueue: + destroy_workqueue(gpu->wq); + +out_thermal: + if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL)) + thermal_cooling_device_unregister(gpu->cooling); + + return ret; } static void etnaviv_gpu_unbind(struct device *dev, struct device *master, @@ -1719,11 +1685,11 @@ static void etnaviv_gpu_unbind(struct device *dev, struct device *master, DBG("%s", dev_name(gpu->dev)); - hangcheck_disable(gpu); - flush_workqueue(gpu->wq); destroy_workqueue(gpu->wq); + etnaviv_sched_fini(gpu); + #ifdef CONFIG_PM pm_runtime_get_sync(gpu->dev); pm_runtime_put_sync_suspend(gpu->dev); @@ -1745,6 +1711,7 @@ static void etnaviv_gpu_unbind(struct device *dev, struct device *master, } gpu->drm = NULL; + idr_destroy(&gpu->fence_idr); if (IS_ENABLED(CONFIG_DRM_ETNAVIV_THERMAL)) thermal_cooling_device_unregister(gpu->cooling); @@ -1762,6 +1729,7 @@ static const struct of_device_id etnaviv_gpu_match[] = { }, { /* sentinel */ } }; +MODULE_DEVICE_TABLE(of, etnaviv_gpu_match); static int etnaviv_gpu_platform_probe(struct platform_device *pdev) { @@ -1775,6 +1743,7 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev) gpu->dev = &pdev->dev; mutex_init(&gpu->lock); + mutex_init(&gpu->fence_idr_lock); /* Map registers: */ gpu->mmio = etnaviv_ioremap(pdev, NULL, dev_name(gpu->dev)); @@ -1796,6 +1765,11 @@ static int etnaviv_gpu_platform_probe(struct platform_device *pdev) } /* Get Clocks: */ + gpu->clk_reg = devm_clk_get(&pdev->dev, "reg"); + DBG("clk_reg: %p", gpu->clk_reg); + if (IS_ERR(gpu->clk_reg)) + gpu->clk_reg = NULL; + gpu->clk_bus = devm_clk_get(&pdev->dev, "bus"); DBG("clk_bus: %p", gpu->clk_bus); if (IS_ERR(gpu->clk_bus)) diff --git a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h index 7623905210dc..3c3005501846 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_gpu.h +++ b/drivers/gpu/drm/etnaviv/etnaviv_gpu.h @@ -38,21 +38,17 @@ struct etnaviv_chip_identity { /* Supported minor feature fields. */ u32 minor_features0; - - /* Supported minor feature 1 fields. */ u32 minor_features1; - - /* Supported minor feature 2 fields. */ u32 minor_features2; - - /* Supported minor feature 3 fields. */ u32 minor_features3; - - /* Supported minor feature 4 fields. */ u32 minor_features4; - - /* Supported minor feature 5 fields. */ u32 minor_features5; + u32 minor_features6; + u32 minor_features7; + u32 minor_features8; + u32 minor_features9; + u32 minor_features10; + u32 minor_features11; /* Number of streams supported. */ u32 stream_count; @@ -88,6 +84,12 @@ struct etnaviv_chip_identity { u8 varyings_count; }; +enum etnaviv_sec_mode { + ETNA_SEC_NONE = 0, + ETNA_SEC_KERNEL, + ETNA_SEC_TZ +}; + struct etnaviv_event { struct dma_fence *fence; struct etnaviv_gem_submit *submit; @@ -106,8 +108,10 @@ struct etnaviv_gpu { struct device *dev; struct mutex lock; struct etnaviv_chip_identity identity; + enum etnaviv_sec_mode sec_mode; struct etnaviv_file_private *lastctx; struct workqueue_struct *wq; + struct drm_gpu_scheduler sched; /* 'ring'-buffer: */ struct etnaviv_cmdbuf buffer; @@ -122,23 +126,18 @@ struct etnaviv_gpu { struct completion event_free; spinlock_t event_spinlock; - /* list of currently in-flight command buffers */ - struct list_head active_submit_list; - u32 idle_mask; /* Fencing support */ + struct mutex fence_idr_lock; + struct idr fence_idr; u32 next_fence; u32 active_fence; u32 completed_fence; - u32 retired_fence; wait_queue_head_t fence_event; u64 fence_context; spinlock_t fence_spinlock; - /* worker for handling active-list retiring: */ - struct work_struct retire_work; - /* worker for handling 'sync' points: */ struct work_struct sync_point_work; int sync_point_event; @@ -151,16 +150,10 @@ struct etnaviv_gpu { /* Power Control: */ struct clk *clk_bus; + struct clk *clk_reg; struct clk *clk_core; struct clk *clk_shader; - /* Hang Detction: */ -#define DRM_ETNAVIV_HANGCHECK_PERIOD 500 /* in ms */ -#define DRM_ETNAVIV_HANGCHECK_JIFFIES msecs_to_jiffies(DRM_ETNAVIV_HANGCHECK_PERIOD) - struct timer_list hangcheck_timer; - u32 hangcheck_fence; - u32 hangcheck_dma_addr; - struct work_struct recover_work; unsigned int freq_scale; unsigned long base_rate_core; unsigned long base_rate_shader; @@ -181,29 +174,22 @@ static inline bool fence_completed(struct etnaviv_gpu *gpu, u32 fence) return fence_after_eq(gpu->completed_fence, fence); } -static inline bool fence_retired(struct etnaviv_gpu *gpu, u32 fence) -{ - return fence_after_eq(gpu->retired_fence, fence); -} - int etnaviv_gpu_get_param(struct etnaviv_gpu *gpu, u32 param, u64 *value); int etnaviv_gpu_init(struct etnaviv_gpu *gpu); +bool etnaviv_fill_identity_from_hwdb(struct etnaviv_gpu *gpu); #ifdef CONFIG_DEBUG_FS int etnaviv_gpu_debugfs(struct etnaviv_gpu *gpu, struct seq_file *m); #endif -int etnaviv_gpu_fence_sync_obj(struct etnaviv_gem_object *etnaviv_obj, - unsigned int context, bool exclusive, bool implicit); - +void etnaviv_gpu_recover_hang(struct etnaviv_gpu *gpu); void etnaviv_gpu_retire(struct etnaviv_gpu *gpu); int etnaviv_gpu_wait_fence_interruptible(struct etnaviv_gpu *gpu, u32 fence, struct timespec *timeout); int etnaviv_gpu_wait_obj_inactive(struct etnaviv_gpu *gpu, struct etnaviv_gem_object *etnaviv_obj, struct timespec *timeout); -int etnaviv_gpu_submit(struct etnaviv_gpu *gpu, - struct etnaviv_gem_submit *submit); +struct dma_fence *etnaviv_gpu_submit(struct etnaviv_gem_submit *submit); int etnaviv_gpu_pm_get_sync(struct etnaviv_gpu *gpu); void etnaviv_gpu_pm_put(struct etnaviv_gpu *gpu); int etnaviv_gpu_wait_idle(struct etnaviv_gpu *gpu, unsigned int timeout_ms); diff --git a/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c b/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c new file mode 100644 index 000000000000..ea08bb38caaf --- /dev/null +++ b/drivers/gpu/drm/etnaviv/etnaviv_hwdb.c @@ -0,0 +1,65 @@ +/* + * Copyright (C) 2018 Etnaviv Project + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include "etnaviv_gpu.h" + +static const struct etnaviv_chip_identity etnaviv_chip_identities[] = { + { + .model = 0x7000, + .revision = 0x6214, + .stream_count = 16, + .register_max = 64, + .thread_count = 1024, + .shader_core_count = 4, + .vertex_cache_size = 16, + .vertex_output_buffer_size = 1024, + .pixel_pipes = 2, + .instruction_count = 512, + .num_constants = 320, + .buffer_size = 0, + .varyings_count = 16, + .features = 0xe0287cad, + .minor_features0 = 0xc1799eff, + .minor_features1 = 0xfefbfad9, + .minor_features2 = 0xeb9d4fbf, + .minor_features3 = 0xedfffced, + .minor_features4 = 0xdb0dafc7, + .minor_features5 = 0xbb5ac333, + .minor_features6 = 0xfc8ee200, + .minor_features7 = 0x03fbfa6f, + .minor_features8 = 0x00ef0ef0, + .minor_features9 = 0x0edbf03c, + .minor_features10 = 0x90044250, + .minor_features11 = 0x00000024, + }, +}; + +bool etnaviv_fill_identity_from_hwdb(struct etnaviv_gpu *gpu) +{ + struct etnaviv_chip_identity *ident = &gpu->identity; + int i; + + for (i = 0; i < ARRAY_SIZE(etnaviv_chip_identities); i++) { + if (etnaviv_chip_identities[i].model == ident->model && + etnaviv_chip_identities[i].revision == ident->revision) { + memcpy(ident, &etnaviv_chip_identities[i], + sizeof(*ident)); + return true; + } + } + + return false; +} diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c index 7a8c94731748..4b9b11ca6f03 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_iommu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu.c @@ -158,7 +158,7 @@ void etnaviv_iommuv1_restore(struct etnaviv_gpu *gpu) gpu_write(gpu, VIVS_MC_MMU_RA_PAGE_TABLE, pgtable); } -const struct etnaviv_iommu_domain_ops etnaviv_iommuv1_ops = { +static const struct etnaviv_iommu_domain_ops etnaviv_iommuv1_ops = { .free = etnaviv_iommuv1_domain_free, .map = etnaviv_iommuv1_map, .unmap = etnaviv_iommuv1_unmap, diff --git a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c index 1e956e266aa3..9752dbd5d28b 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_iommu_v2.c @@ -40,6 +40,9 @@ struct etnaviv_iommuv2_domain { struct etnaviv_iommu_domain base; + /* P(age) T(able) A(rray) */ + u64 *pta_cpu; + dma_addr_t pta_dma; /* M(aster) TLB aka first level pagetable */ u32 *mtlb_cpu; dma_addr_t mtlb_dma; @@ -114,6 +117,15 @@ static int etnaviv_iommuv2_init(struct etnaviv_iommuv2_domain *etnaviv_domain) for (i = 0; i < SZ_4K / 4; i++) *p++ = 0xdead55aa; + etnaviv_domain->pta_cpu = dma_alloc_coherent(etnaviv_domain->base.dev, + SZ_4K, + &etnaviv_domain->pta_dma, + GFP_KERNEL); + if (!etnaviv_domain->pta_cpu) { + ret = -ENOMEM; + goto fail_mem; + } + etnaviv_domain->mtlb_cpu = dma_alloc_coherent(etnaviv_domain->base.dev, SZ_4K, &etnaviv_domain->mtlb_dma, @@ -150,6 +162,11 @@ fail_mem: etnaviv_domain->base.bad_page_cpu, etnaviv_domain->base.bad_page_dma); + if (etnaviv_domain->pta_cpu) + dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, + etnaviv_domain->pta_cpu, + etnaviv_domain->pta_dma); + if (etnaviv_domain->mtlb_cpu) dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, etnaviv_domain->mtlb_cpu, @@ -176,6 +193,10 @@ static void etnaviv_iommuv2_domain_free(struct etnaviv_iommu_domain *domain) etnaviv_domain->base.bad_page_dma); dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, + etnaviv_domain->pta_cpu, + etnaviv_domain->pta_dma); + + dma_free_coherent(etnaviv_domain->base.dev, SZ_4K, etnaviv_domain->mtlb_cpu, etnaviv_domain->mtlb_dma); @@ -216,7 +237,7 @@ static void etnaviv_iommuv2_dump(struct etnaviv_iommu_domain *domain, void *buf) memcpy(buf, etnaviv_domain->stlb_cpu[i], SZ_4K); } -void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) +static void etnaviv_iommuv2_restore_nonsec(struct etnaviv_gpu *gpu) { struct etnaviv_iommuv2_domain *etnaviv_domain = to_etnaviv_domain(gpu->mmu->domain); @@ -236,7 +257,60 @@ void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) gpu_write(gpu, VIVS_MMUv2_CONTROL, VIVS_MMUv2_CONTROL_ENABLE); } -const struct etnaviv_iommu_domain_ops etnaviv_iommuv2_ops = { +static void etnaviv_iommuv2_restore_sec(struct etnaviv_gpu *gpu) +{ + struct etnaviv_iommuv2_domain *etnaviv_domain = + to_etnaviv_domain(gpu->mmu->domain); + u16 prefetch; + + /* If the MMU is already enabled the state is still there. */ + if (gpu_read(gpu, VIVS_MMUv2_SEC_CONTROL) & VIVS_MMUv2_SEC_CONTROL_ENABLE) + return; + + gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_LOW, + lower_32_bits(etnaviv_domain->pta_dma)); + gpu_write(gpu, VIVS_MMUv2_PTA_ADDRESS_HIGH, + upper_32_bits(etnaviv_domain->pta_dma)); + gpu_write(gpu, VIVS_MMUv2_PTA_CONTROL, VIVS_MMUv2_PTA_CONTROL_ENABLE); + + gpu_write(gpu, VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW, + lower_32_bits(etnaviv_domain->base.bad_page_dma)); + gpu_write(gpu, VIVS_MMUv2_SEC_SAFE_ADDR_LOW, + lower_32_bits(etnaviv_domain->base.bad_page_dma)); + gpu_write(gpu, VIVS_MMUv2_SAFE_ADDRESS_CONFIG, + VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH( + upper_32_bits(etnaviv_domain->base.bad_page_dma)) | + VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH( + upper_32_bits(etnaviv_domain->base.bad_page_dma))); + + etnaviv_domain->pta_cpu[0] = etnaviv_domain->mtlb_dma | + VIVS_MMUv2_CONFIGURATION_MODE_MODE4_K; + + /* trigger a PTA load through the FE */ + prefetch = etnaviv_buffer_config_pta(gpu); + etnaviv_gpu_start_fe(gpu, (u32)etnaviv_cmdbuf_get_pa(&gpu->buffer), + prefetch); + etnaviv_gpu_wait_idle(gpu, 100); + + gpu_write(gpu, VIVS_MMUv2_SEC_CONTROL, VIVS_MMUv2_SEC_CONTROL_ENABLE); +} + +void etnaviv_iommuv2_restore(struct etnaviv_gpu *gpu) +{ + switch (gpu->sec_mode) { + case ETNA_SEC_NONE: + etnaviv_iommuv2_restore_nonsec(gpu); + break; + case ETNA_SEC_KERNEL: + etnaviv_iommuv2_restore_sec(gpu); + break; + default: + WARN(1, "unhandled GPU security mode\n"); + break; + } +} + +static const struct etnaviv_iommu_domain_ops etnaviv_iommuv2_ops = { .free = etnaviv_iommuv2_domain_free, .map = etnaviv_iommuv2_map, .unmap = etnaviv_iommuv2_unmap, diff --git a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c index d113fe06e6b5..49e049713a52 100644 --- a/drivers/gpu/drm/etnaviv/etnaviv_mmu.c +++ b/drivers/gpu/drm/etnaviv/etnaviv_mmu.c @@ -29,7 +29,7 @@ static void etnaviv_domain_unmap(struct etnaviv_iommu_domain *domain, size_t pgsize = SZ_4K; if (!IS_ALIGNED(iova | size, pgsize)) { - pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%x\n", + pr_err("unaligned: iova 0x%lx size 0x%zx min_pagesz 0x%zx\n", iova, size, pgsize); return; } @@ -54,7 +54,7 @@ static int etnaviv_domain_map(struct etnaviv_iommu_domain *domain, int ret = 0; if (!IS_ALIGNED(iova | paddr | size, pgsize)) { - pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%x\n", + pr_err("unaligned: iova 0x%lx pa %pa size 0x%zx min_pagesz 0x%zx\n", iova, &paddr, size, pgsize); return -EINVAL; } diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.c b/drivers/gpu/drm/etnaviv/etnaviv_sched.c new file mode 100644 index 000000000000..6cf0775dbcd7 --- /dev/null +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.c @@ -0,0 +1,170 @@ +/* + * Copyright (C) 2017 Etnaviv Project + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#include <linux/kthread.h> + +#include "etnaviv_drv.h" +#include "etnaviv_dump.h" +#include "etnaviv_gem.h" +#include "etnaviv_gpu.h" +#include "etnaviv_sched.h" + +static int etnaviv_job_hang_limit = 0; +module_param_named(job_hang_limit, etnaviv_job_hang_limit, int , 0444); +static int etnaviv_hw_jobs_limit = 4; +module_param_named(hw_job_limit, etnaviv_hw_jobs_limit, int , 0444); + +static struct dma_fence * +etnaviv_sched_dependency(struct drm_sched_job *sched_job, + struct drm_sched_entity *entity) +{ + struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job); + struct dma_fence *fence; + int i; + + if (unlikely(submit->in_fence)) { + fence = submit->in_fence; + submit->in_fence = NULL; + + if (!dma_fence_is_signaled(fence)) + return fence; + + dma_fence_put(fence); + } + + for (i = 0; i < submit->nr_bos; i++) { + struct etnaviv_gem_submit_bo *bo = &submit->bos[i]; + int j; + + if (bo->excl) { + fence = bo->excl; + bo->excl = NULL; + + if (!dma_fence_is_signaled(fence)) + return fence; + + dma_fence_put(fence); + } + + for (j = 0; j < bo->nr_shared; j++) { + if (!bo->shared[j]) + continue; + + fence = bo->shared[j]; + bo->shared[j] = NULL; + + if (!dma_fence_is_signaled(fence)) + return fence; + + dma_fence_put(fence); + } + kfree(bo->shared); + bo->nr_shared = 0; + bo->shared = NULL; + } + + return NULL; +} + +static struct dma_fence *etnaviv_sched_run_job(struct drm_sched_job *sched_job) +{ + struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job); + struct dma_fence *fence = NULL; + + if (likely(!sched_job->s_fence->finished.error)) + fence = etnaviv_gpu_submit(submit); + else + dev_dbg(submit->gpu->dev, "skipping bad job\n"); + + return fence; +} + +static void etnaviv_sched_timedout_job(struct drm_sched_job *sched_job) +{ + struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job); + struct etnaviv_gpu *gpu = submit->gpu; + + /* block scheduler */ + kthread_park(gpu->sched.thread); + drm_sched_hw_job_reset(&gpu->sched, sched_job); + + /* get the GPU back into the init state */ + etnaviv_core_dump(gpu); + etnaviv_gpu_recover_hang(gpu); + + /* restart scheduler after GPU is usable again */ + drm_sched_job_recovery(&gpu->sched); + kthread_unpark(gpu->sched.thread); +} + +static void etnaviv_sched_free_job(struct drm_sched_job *sched_job) +{ + struct etnaviv_gem_submit *submit = to_etnaviv_submit(sched_job); + + etnaviv_submit_put(submit); +} + +static const struct drm_sched_backend_ops etnaviv_sched_ops = { + .dependency = etnaviv_sched_dependency, + .run_job = etnaviv_sched_run_job, + .timedout_job = etnaviv_sched_timedout_job, + .free_job = etnaviv_sched_free_job, +}; + +int etnaviv_sched_push_job(struct drm_sched_entity *sched_entity, + struct etnaviv_gem_submit *submit) +{ + int ret; + + ret = drm_sched_job_init(&submit->sched_job, &submit->gpu->sched, + sched_entity, submit->cmdbuf.ctx); + if (ret) + return ret; + + submit->out_fence = dma_fence_get(&submit->sched_job.s_fence->finished); + mutex_lock(&submit->gpu->fence_idr_lock); + submit->out_fence_id = idr_alloc_cyclic(&submit->gpu->fence_idr, + submit->out_fence, 0, + INT_MAX, GFP_KERNEL); + mutex_unlock(&submit->gpu->fence_idr_lock); + if (submit->out_fence_id < 0) + return -ENOMEM; + + /* the scheduler holds on to the job now */ + kref_get(&submit->refcount); + + drm_sched_entity_push_job(&submit->sched_job, sched_entity); + + return 0; +} + +int etnaviv_sched_init(struct etnaviv_gpu *gpu) +{ + int ret; + + ret = drm_sched_init(&gpu->sched, &etnaviv_sched_ops, + etnaviv_hw_jobs_limit, etnaviv_job_hang_limit, + msecs_to_jiffies(500), dev_name(gpu->dev)); + if (ret) + return ret; + + return 0; +} + +void etnaviv_sched_fini(struct etnaviv_gpu *gpu) +{ + drm_sched_fini(&gpu->sched); +} diff --git a/drivers/gpu/drm/etnaviv/etnaviv_sched.h b/drivers/gpu/drm/etnaviv/etnaviv_sched.h new file mode 100644 index 000000000000..097635fa78ae --- /dev/null +++ b/drivers/gpu/drm/etnaviv/etnaviv_sched.h @@ -0,0 +1,35 @@ +/* + * Copyright (C) 2017 Etnaviv Project + * + * This program is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 as published by + * the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for + * more details. + * + * You should have received a copy of the GNU General Public License along with + * this program. If not, see <http://www.gnu.org/licenses/>. + */ + +#ifndef __ETNAVIV_SCHED_H__ +#define __ETNAVIV_SCHED_H__ + +#include <drm/gpu_scheduler.h> + +struct etnaviv_gpu; + +static inline +struct etnaviv_gem_submit *to_etnaviv_submit(struct drm_sched_job *sched_job) +{ + return container_of(sched_job, struct etnaviv_gem_submit, sched_job); +} + +int etnaviv_sched_init(struct etnaviv_gpu *gpu); +void etnaviv_sched_fini(struct etnaviv_gpu *gpu); +int etnaviv_sched_push_job(struct drm_sched_entity *sched_entity, + struct etnaviv_gem_submit *submit); + +#endif /* __ETNAVIV_SCHED_H__ */ diff --git a/drivers/gpu/drm/etnaviv/state.xml.h b/drivers/gpu/drm/etnaviv/state.xml.h index c27c1484cfa9..421cb7cc0053 100644 --- a/drivers/gpu/drm/etnaviv/state.xml.h +++ b/drivers/gpu/drm/etnaviv/state.xml.h @@ -1,4 +1,3 @@ -/* SPDX-License-Identifier: GPL-2.0 */ #ifndef STATE_XML #define STATE_XML @@ -9,14 +8,40 @@ http://0x04.net/cgit/index.cgi/rules-ng-ng git clone git://0x04.net/rules-ng-ng The rules-ng-ng source files this header was generated from are: -- state.xml ( 18882 bytes, from 2015-03-25 11:42:32) -- common.xml ( 18437 bytes, from 2015-03-25 11:27:41) -- state_hi.xml ( 23420 bytes, from 2015-03-25 11:47:21) -- state_2d.xml ( 51549 bytes, from 2015-03-25 11:25:06) -- state_3d.xml ( 54600 bytes, from 2015-03-25 11:25:19) -- state_vg.xml ( 5973 bytes, from 2015-03-25 11:26:01) - -Copyright (C) 2015 +- state.xml ( 26087 bytes, from 2017-12-18 16:51:59) +- common.xml ( 35468 bytes, from 2018-01-22 13:48:54) +- common_3d.xml ( 14615 bytes, from 2017-12-18 16:51:59) +- state_hi.xml ( 30232 bytes, from 2018-02-15 15:48:01) +- copyright.xml ( 1597 bytes, from 2016-12-08 16:37:56) +- state_2d.xml ( 51552 bytes, from 2016-12-08 16:37:56) +- state_3d.xml ( 79992 bytes, from 2017-12-18 16:51:59) +- state_blt.xml ( 13405 bytes, from 2017-12-18 16:51:59) +- state_vg.xml ( 5975 bytes, from 2016-12-08 16:37:56) + +Copyright (C) 2012-2017 by the following authors: +- Wladimir J. van der Laan <laanwj@gmail.com> +- Christian Gmeiner <christian.gmeiner@gmail.com> +- Lucas Stach <l.stach@pengutronix.de> +- Russell King <rmk@arm.linux.org.uk> + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sub license, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the +next paragraph) shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. */ @@ -24,9 +49,25 @@ Copyright (C) 2015 #define VARYING_COMPONENT_USE_USED 0x00000001 #define VARYING_COMPONENT_USE_POINTCOORD_X 0x00000002 #define VARYING_COMPONENT_USE_POINTCOORD_Y 0x00000003 +#define FE_DATA_TYPE_BYTE 0x00000000 +#define FE_DATA_TYPE_UNSIGNED_BYTE 0x00000001 +#define FE_DATA_TYPE_SHORT 0x00000002 +#define FE_DATA_TYPE_UNSIGNED_SHORT 0x00000003 +#define FE_DATA_TYPE_INT 0x00000004 +#define FE_DATA_TYPE_UNSIGNED_INT 0x00000005 +#define FE_DATA_TYPE_FLOAT 0x00000008 +#define FE_DATA_TYPE_HALF_FLOAT 0x00000009 +#define FE_DATA_TYPE_FIXED 0x0000000b +#define FE_DATA_TYPE_INT_10_10_10_2 0x0000000c +#define FE_DATA_TYPE_UNSIGNED_INT_10_10_10_2 0x0000000d +#define FE_DATA_TYPE_BYTE_I 0x0000000e +#define FE_DATA_TYPE_SHORT_I 0x0000000f #define FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE__MASK 0x000000ff #define FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE__SHIFT 0 #define FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE(x) (((x) << FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE__SHIFT) & FE_VERTEX_STREAM_CONTROL_VERTEX_STRIDE__MASK) +#define FE_VERTEX_STREAM_CONTROL_VERTEX_DIVISOR__MASK 0x00ff0000 +#define FE_VERTEX_STREAM_CONTROL_VERTEX_DIVISOR__SHIFT 16 +#define FE_VERTEX_STREAM_CONTROL_VERTEX_DIVISOR(x) (((x) << FE_VERTEX_STREAM_CONTROL_VERTEX_DIVISOR__SHIFT) & FE_VERTEX_STREAM_CONTROL_VERTEX_DIVISOR__MASK) #define VIVS_FE 0x00000000 #define VIVS_FE_VERTEX_ELEMENT_CONFIG(i0) (0x00000600 + 0x4*(i0)) @@ -34,17 +75,7 @@ Copyright (C) 2015 #define VIVS_FE_VERTEX_ELEMENT_CONFIG__LEN 0x00000010 #define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE__MASK 0x0000000f #define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE__SHIFT 0 -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_BYTE 0x00000000 -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_UNSIGNED_BYTE 0x00000001 -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_SHORT 0x00000002 -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_UNSIGNED_SHORT 0x00000003 -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_INT 0x00000004 -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_UNSIGNED_INT 0x00000005 -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_FLOAT 0x00000008 -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_HALF_FLOAT 0x00000009 -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_FIXED 0x0000000b -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_INT_10_10_10_2 0x0000000c -#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE_UNSIGNED_INT_10_10_10_2 0x0000000d +#define VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE(x) (((x) << VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE__SHIFT) & VIVS_FE_VERTEX_ELEMENT_CONFIG_TYPE__MASK) #define VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN__MASK 0x00000030 #define VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN__SHIFT 4 #define VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN(x) (((x) << VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN__SHIFT) & VIVS_FE_VERTEX_ELEMENT_CONFIG_ENDIAN__MASK) @@ -76,6 +107,7 @@ Copyright (C) 2015 #define VIVS_FE_INDEX_STREAM_CONTROL_TYPE_UNSIGNED_CHAR 0x00000000 #define VIVS_FE_INDEX_STREAM_CONTROL_TYPE_UNSIGNED_SHORT 0x00000001 #define VIVS_FE_INDEX_STREAM_CONTROL_TYPE_UNSIGNED_INT 0x00000002 +#define VIVS_FE_INDEX_STREAM_CONTROL_PRIMITIVE_RESTART 0x00000100 #define VIVS_FE_VERTEX_STREAM_BASE_ADDR 0x0000064c @@ -151,6 +183,8 @@ Copyright (C) 2015 #define VIVS_FE_AUTO_FLUSH 0x00000670 +#define VIVS_FE_PRIMITIVE_RESTART_INDEX 0x00000674 + #define VIVS_FE_UNK00678 0x00000678 #define VIVS_FE_UNK0067C 0x0000067c @@ -163,17 +197,40 @@ Copyright (C) 2015 #define VIVS_FE_VERTEX_STREAMS_CONTROL(i0) (0x000006a0 + 0x4*(i0)) -#define VIVS_FE_UNK00700(i0) (0x00000700 + 0x4*(i0)) -#define VIVS_FE_UNK00700__ESIZE 0x00000004 -#define VIVS_FE_UNK00700__LEN 0x00000010 +#define VIVS_FE_GENERIC_ATTRIB(i0) (0x00000000 + 0x4*(i0)) +#define VIVS_FE_GENERIC_ATTRIB__ESIZE 0x00000004 +#define VIVS_FE_GENERIC_ATTRIB__LEN 0x00000010 + +#define VIVS_FE_GENERIC_ATTRIB_UNK006C0(i0) (0x000006c0 + 0x4*(i0)) + +#define VIVS_FE_GENERIC_ATTRIB_UNK00700(i0) (0x00000700 + 0x4*(i0)) + +#define VIVS_FE_GENERIC_ATTRIB_UNK00740(i0) (0x00000740 + 0x4*(i0)) + +#define VIVS_FE_GENERIC_ATTRIB_SCALE(i0) (0x00000780 + 0x4*(i0)) + +#define VIVS_FE_HALTI5_UNK007C4 0x000007c4 + +#define VIVS_FE_HALTI5_UNK007D0(i0) (0x000007d0 + 0x4*(i0)) +#define VIVS_FE_HALTI5_UNK007D0__ESIZE 0x00000004 +#define VIVS_FE_HALTI5_UNK007D0__LEN 0x00000002 + +#define VIVS_FE_HALTI5_UNK007D8 0x000007d8 + +#define VIVS_FE_DESC_START 0x000007dc + +#define VIVS_FE_DESC_END 0x000007e0 + +#define VIVS_FE_DESC_AVAIL 0x000007e4 +#define VIVS_FE_DESC_AVAIL_COUNT__MASK 0x0000007f +#define VIVS_FE_DESC_AVAIL_COUNT__SHIFT 0 +#define VIVS_FE_DESC_AVAIL_COUNT(x) (((x) << VIVS_FE_DESC_AVAIL_COUNT__SHIFT) & VIVS_FE_DESC_AVAIL_COUNT__MASK) + +#define VIVS_FE_FENCE_WAIT_DATA_LOW 0x000007e8 -#define VIVS_FE_UNK00740(i0) (0x00000740 + 0x4*(i0)) -#define VIVS_FE_UNK00740__ESIZE 0x00000004 -#define VIVS_FE_UNK00740__LEN 0x00000010 +#define VIVS_FE_FENCE_WAIT_DATA_HIGH 0x000007f4 -#define VIVS_FE_UNK00780(i0) (0x00000780 + 0x4*(i0)) -#define VIVS_FE_UNK00780__ESIZE 0x00000004 -#define VIVS_FE_UNK00780__LEN 0x00000010 +#define VIVS_FE_ROBUSTNESS_UNK007F8 0x000007f8 #define VIVS_GL 0x00000000 @@ -188,6 +245,7 @@ Copyright (C) 2015 #define VIVS_GL_EVENT_EVENT_ID(x) (((x) << VIVS_GL_EVENT_EVENT_ID__SHIFT) & VIVS_GL_EVENT_EVENT_ID__MASK) #define VIVS_GL_EVENT_FROM_FE 0x00000020 #define VIVS_GL_EVENT_FROM_PE 0x00000040 +#define VIVS_GL_EVENT_FROM_BLT 0x00000080 #define VIVS_GL_EVENT_SOURCE__MASK 0x00001f00 #define VIVS_GL_EVENT_SOURCE__SHIFT 8 #define VIVS_GL_EVENT_SOURCE(x) (((x) << VIVS_GL_EVENT_SOURCE__SHIFT) & VIVS_GL_EVENT_SOURCE__MASK) @@ -199,6 +257,9 @@ Copyright (C) 2015 #define VIVS_GL_SEMAPHORE_TOKEN_TO__MASK 0x00001f00 #define VIVS_GL_SEMAPHORE_TOKEN_TO__SHIFT 8 #define VIVS_GL_SEMAPHORE_TOKEN_TO(x) (((x) << VIVS_GL_SEMAPHORE_TOKEN_TO__SHIFT) & VIVS_GL_SEMAPHORE_TOKEN_TO__MASK) +#define VIVS_GL_SEMAPHORE_TOKEN_UNK28__MASK 0x30000000 +#define VIVS_GL_SEMAPHORE_TOKEN_UNK28__SHIFT 28 +#define VIVS_GL_SEMAPHORE_TOKEN_UNK28(x) (((x) << VIVS_GL_SEMAPHORE_TOKEN_UNK28__SHIFT) & VIVS_GL_SEMAPHORE_TOKEN_UNK28__MASK) #define VIVS_GL_FLUSH_CACHE 0x0000380c #define VIVS_GL_FLUSH_CACHE_DEPTH 0x00000001 @@ -208,6 +269,10 @@ Copyright (C) 2015 #define VIVS_GL_FLUSH_CACHE_TEXTUREVS 0x00000010 #define VIVS_GL_FLUSH_CACHE_SHADER_L1 0x00000020 #define VIVS_GL_FLUSH_CACHE_SHADER_L2 0x00000040 +#define VIVS_GL_FLUSH_CACHE_UNK10 0x00000400 +#define VIVS_GL_FLUSH_CACHE_UNK11 0x00000800 +#define VIVS_GL_FLUSH_CACHE_DESCRIPTOR_UNK12 0x00001000 +#define VIVS_GL_FLUSH_CACHE_DESCRIPTOR_UNK13 0x00002000 #define VIVS_GL_FLUSH_MMU 0x00003810 #define VIVS_GL_FLUSH_MMU_FLUSH_FEMMU 0x00000001 @@ -244,30 +309,8 @@ Copyright (C) 2015 #define VIVS_GL_VARYING_TOTAL_COMPONENTS_NUM(x) (((x) << VIVS_GL_VARYING_TOTAL_COMPONENTS_NUM__SHIFT) & VIVS_GL_VARYING_TOTAL_COMPONENTS_NUM__MASK) #define VIVS_GL_VARYING_NUM_COMPONENTS 0x00003820 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR0__MASK 0x00000007 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR0__SHIFT 0 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR0(x) (((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR0__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR0__MASK) -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR1__MASK 0x00000070 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR1__SHIFT 4 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR1(x) (((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR1__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR1__MASK) -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR2__MASK 0x00000700 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR2__SHIFT 8 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR2(x) (((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR2__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR2__MASK) -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR3__MASK 0x00007000 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR3__SHIFT 12 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR3(x) (((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR3__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR3__MASK) -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR4__MASK 0x00070000 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR4__SHIFT 16 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR4(x) (((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR4__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR4__MASK) -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR5__MASK 0x00700000 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR5__SHIFT 20 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR5(x) (((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR5__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR5__MASK) -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR6__MASK 0x07000000 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR6__SHIFT 24 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR6(x) (((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR6__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR6__MASK) -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR7__MASK 0x70000000 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR7__SHIFT 28 -#define VIVS_GL_VARYING_NUM_COMPONENTS_VAR7(x) (((x) << VIVS_GL_VARYING_NUM_COMPONENTS_VAR7__SHIFT) & VIVS_GL_VARYING_NUM_COMPONENTS_VAR7__MASK) + +#define VIVS_GL_OCCLUSION_QUERY_ADDR 0x00003824 #define VIVS_GL_VARYING_COMPONENT_USE(i0) (0x00003828 + 0x4*(i0)) #define VIVS_GL_VARYING_COMPONENT_USE__ESIZE 0x00000004 @@ -321,6 +364,10 @@ Copyright (C) 2015 #define VIVS_GL_VARYING_COMPONENT_USE_COMP15__SHIFT 30 #define VIVS_GL_VARYING_COMPONENT_USE_COMP15(x) (((x) << VIVS_GL_VARYING_COMPONENT_USE_COMP15__SHIFT) & VIVS_GL_VARYING_COMPONENT_USE_COMP15__MASK) +#define VIVS_GL_UNK0382C 0x0000382c + +#define VIVS_GL_OCCLUSION_QUERY_CONTROL 0x00003830 + #define VIVS_GL_UNK03834 0x00003834 #define VIVS_GL_UNK03838 0x00003838 @@ -332,8 +379,58 @@ Copyright (C) 2015 #define VIVS_GL_CONTEXT_POINTER 0x00003850 +#define VIVS_GL_UNK03854 0x00003854 + +#define VIVS_GL_BUG_FIXES 0x00003860 + +#define VIVS_GL_FENCE_OUT_ADDRESS 0x00003868 + +#define VIVS_GL_FENCE_OUT_DATA_LOW 0x0000386c + +#define VIVS_GL_HALTI5_UNK03884 0x00003884 + +#define VIVS_GL_HALTI5_SH_SPECIALS 0x00003888 +#define VIVS_GL_HALTI5_SH_SPECIALS_VS_PSIZE_OUT__MASK 0x0000007f +#define VIVS_GL_HALTI5_SH_SPECIALS_VS_PSIZE_OUT__SHIFT 0 +#define VIVS_GL_HALTI5_SH_SPECIALS_VS_PSIZE_OUT(x) (((x) << VIVS_GL_HALTI5_SH_SPECIALS_VS_PSIZE_OUT__SHIFT) & VIVS_GL_HALTI5_SH_SPECIALS_VS_PSIZE_OUT__MASK) +#define VIVS_GL_HALTI5_SH_SPECIALS_PS_PCOORD_IN__MASK 0x00007f00 +#define VIVS_GL_HALTI5_SH_SPECIALS_PS_PCOORD_IN__SHIFT 8 +#define VIVS_GL_HALTI5_SH_SPECIALS_PS_PCOORD_IN(x) (((x) << VIVS_GL_HALTI5_SH_SPECIALS_PS_PCOORD_IN__SHIFT) & VIVS_GL_HALTI5_SH_SPECIALS_PS_PCOORD_IN__MASK) +#define VIVS_GL_HALTI5_SH_SPECIALS_UNK16__MASK 0x007f0000 +#define VIVS_GL_HALTI5_SH_SPECIALS_UNK16__SHIFT 16 +#define VIVS_GL_HALTI5_SH_SPECIALS_UNK16(x) (((x) << VIVS_GL_HALTI5_SH_SPECIALS_UNK16__SHIFT) & VIVS_GL_HALTI5_SH_SPECIALS_UNK16__MASK) +#define VIVS_GL_HALTI5_SH_SPECIALS_UNK24__MASK 0xff000000 +#define VIVS_GL_HALTI5_SH_SPECIALS_UNK24__SHIFT 24 +#define VIVS_GL_HALTI5_SH_SPECIALS_UNK24(x) (((x) << VIVS_GL_HALTI5_SH_SPECIALS_UNK24__SHIFT) & VIVS_GL_HALTI5_SH_SPECIALS_UNK24__MASK) + +#define VIVS_GL_GS_UNK0388C 0x0000388c + +#define VIVS_GL_FENCE_OUT_DATA_HIGH 0x00003898 + +#define VIVS_GL_SHADER_INDEX 0x0000389c + +#define VIVS_GL_GS_UNK038A0(i0) (0x000038a0 + 0x4*(i0)) +#define VIVS_GL_GS_UNK038A0__ESIZE 0x00000004 +#define VIVS_GL_GS_UNK038A0__LEN 0x00000008 + +#define VIVS_GL_HALTI5_UNK038C0(i0) (0x000038c0 + 0x4*(i0)) +#define VIVS_GL_HALTI5_UNK038C0__ESIZE 0x00000004 +#define VIVS_GL_HALTI5_UNK038C0__LEN 0x00000010 + +#define VIVS_GL_SECURITY_UNK3900 0x00003900 + +#define VIVS_GL_SECURITY_UNK3904 0x00003904 + #define VIVS_GL_UNK03A00 0x00003a00 +#define VIVS_GL_UNK03A04 0x00003a04 + +#define VIVS_GL_UNK03A08 0x00003a08 + +#define VIVS_GL_UNK03A0C 0x00003a0c + +#define VIVS_GL_UNK03A10 0x00003a10 + #define VIVS_GL_STALL_TOKEN 0x00003c00 #define VIVS_GL_STALL_TOKEN_FROM__MASK 0x0000001f #define VIVS_GL_STALL_TOKEN_FROM__SHIFT 0 @@ -344,6 +441,59 @@ Copyright (C) 2015 #define VIVS_GL_STALL_TOKEN_FLIP0 0x40000000 #define VIVS_GL_STALL_TOKEN_FLIP1 0x80000000 +#define VIVS_NFE 0x00000000 + +#define VIVS_NFE_VERTEX_STREAMS(i0) (0x00000000 + 0x4*(i0)) +#define VIVS_NFE_VERTEX_STREAMS__ESIZE 0x00000004 +#define VIVS_NFE_VERTEX_STREAMS__LEN 0x00000010 + +#define VIVS_NFE_VERTEX_STREAMS_BASE_ADDR(i0) (0x00014600 + 0x4*(i0)) + +#define VIVS_NFE_VERTEX_STREAMS_CONTROL(i0) (0x00014640 + 0x4*(i0)) + +#define VIVS_NFE_VERTEX_STREAMS_UNK14680(i0) (0x00014680 + 0x4*(i0)) + +#define VIVS_NFE_VERTEX_STREAMS_ROBUSTNESS_UNK146C0(i0) (0x000146c0 + 0x4*(i0)) + +#define VIVS_NFE_GENERIC_ATTRIB(i0) (0x00000000 + 0x4*(i0)) +#define VIVS_NFE_GENERIC_ATTRIB__ESIZE 0x00000004 +#define VIVS_NFE_GENERIC_ATTRIB__LEN 0x00000020 + +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0(i0) (0x00017800 + 0x4*(i0)) +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_TYPE__MASK 0x0000000f +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_TYPE__SHIFT 0 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_TYPE(x) (((x) << VIVS_NFE_GENERIC_ATTRIB_CONFIG0_TYPE__SHIFT) & VIVS_NFE_GENERIC_ATTRIB_CONFIG0_TYPE__MASK) +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_ENDIAN__MASK 0x00000030 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_ENDIAN__SHIFT 4 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_ENDIAN(x) (((x) << VIVS_NFE_GENERIC_ATTRIB_CONFIG0_ENDIAN__SHIFT) & VIVS_NFE_GENERIC_ATTRIB_CONFIG0_ENDIAN__MASK) +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_STREAM__MASK 0x00000700 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_STREAM__SHIFT 8 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_STREAM(x) (((x) << VIVS_NFE_GENERIC_ATTRIB_CONFIG0_STREAM__SHIFT) & VIVS_NFE_GENERIC_ATTRIB_CONFIG0_STREAM__MASK) +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_NUM__MASK 0x00003000 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_NUM__SHIFT 12 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_NUM(x) (((x) << VIVS_NFE_GENERIC_ATTRIB_CONFIG0_NUM__SHIFT) & VIVS_NFE_GENERIC_ATTRIB_CONFIG0_NUM__MASK) +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_NORMALIZE__MASK 0x0000c000 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_NORMALIZE__SHIFT 14 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_NORMALIZE_OFF 0x00000000 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_NORMALIZE_ON 0x00008000 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_START__MASK 0x00ff0000 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_START__SHIFT 16 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG0_START(x) (((x) << VIVS_NFE_GENERIC_ATTRIB_CONFIG0_START__SHIFT) & VIVS_NFE_GENERIC_ATTRIB_CONFIG0_START__MASK) + +#define VIVS_NFE_GENERIC_ATTRIB_UNK17880(i0) (0x00017880 + 0x4*(i0)) + +#define VIVS_NFE_GENERIC_ATTRIB_UNK17900(i0) (0x00017900 + 0x4*(i0)) + +#define VIVS_NFE_GENERIC_ATTRIB_UNK17980(i0) (0x00017980 + 0x4*(i0)) + +#define VIVS_NFE_GENERIC_ATTRIB_SCALE(i0) (0x00017a00 + 0x4*(i0)) + +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG1(i0) (0x00017a80 + 0x4*(i0)) +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG1_END__MASK 0x000000ff +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG1_END__SHIFT 0 +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG1_END(x) (((x) << VIVS_NFE_GENERIC_ATTRIB_CONFIG1_END__SHIFT) & VIVS_NFE_GENERIC_ATTRIB_CONFIG1_END__MASK) +#define VIVS_NFE_GENERIC_ATTRIB_CONFIG1_NONCONSECUTIVE 0x00000800 + #define VIVS_DUMMY 0x00000000 #define VIVS_DUMMY_DUMMY 0x0003fffc diff --git a/drivers/gpu/drm/etnaviv/state_3d.xml.h b/drivers/gpu/drm/etnaviv/state_3d.xml.h index 73a97d35c51b..ebbd4fcf3096 100644 --- a/drivers/gpu/drm/etnaviv/state_3d.xml.h +++ b/drivers/gpu/drm/etnaviv/state_3d.xml.h @@ -7,4 +7,9 @@ #define VIVS_TS_FLUSH_CACHE 0x00001650 #define VIVS_TS_FLUSH_CACHE_FLUSH 0x00000001 +#define VIVS_NTE_DESCRIPTOR_FLUSH 0x00014c44 +#define VIVS_NTE_DESCRIPTOR_FLUSH_UNK28__MASK 0xf0000000 +#define VIVS_NTE_DESCRIPTOR_FLUSH_UNK28__SHIFT 28 +#define VIVS_NTE_DESCRIPTOR_FLUSH_UNK28(x) (((x) << VIVS_NTE_DESCRIPTOR_FLUSH_UNK28__SHIFT) & VIVS_NTE_DESCRIPTOR_FLUSH_UNK28__MASK) + #endif /* STATE_3D_XML */ diff --git a/drivers/gpu/drm/etnaviv/state_blt.xml.h b/drivers/gpu/drm/etnaviv/state_blt.xml.h new file mode 100644 index 000000000000..daae55995def --- /dev/null +++ b/drivers/gpu/drm/etnaviv/state_blt.xml.h @@ -0,0 +1,52 @@ +#ifndef STATE_BLT_XML +#define STATE_BLT_XML + +/* Autogenerated file, DO NOT EDIT manually! + +This file was generated by the rules-ng-ng headergen tool in this git repository: +http://0x04.net/cgit/index.cgi/rules-ng-ng +git clone git://0x04.net/rules-ng-ng + +The rules-ng-ng source files this header was generated from are: +- state.xml ( 26087 bytes, from 2017-12-18 16:51:59) +- common.xml ( 35468 bytes, from 2018-01-22 13:48:54) +- common_3d.xml ( 14615 bytes, from 2017-12-18 16:51:59) +- state_hi.xml ( 30232 bytes, from 2018-02-15 15:48:01) +- copyright.xml ( 1597 bytes, from 2016-12-08 16:37:56) +- state_2d.xml ( 51552 bytes, from 2016-12-08 16:37:56) +- state_3d.xml ( 79992 bytes, from 2017-12-18 16:51:59) +- state_blt.xml ( 13405 bytes, from 2017-12-18 16:51:59) +- state_vg.xml ( 5975 bytes, from 2016-12-08 16:37:56) + +Copyright (C) 2012-2017 by the following authors: +- Wladimir J. van der Laan <laanwj@gmail.com> +- Christian Gmeiner <christian.gmeiner@gmail.com> +- Lucas Stach <l.stach@pengutronix.de> +- Russell King <rmk@arm.linux.org.uk> + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sub license, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the +next paragraph) shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. +*/ + +/* This is a cut-down version of the state_blt.xml.h file */ + +#define VIVS_BLT_ENABLE 0x000140b8 +#define VIVS_BLT_ENABLE_ENABLE 0x00000001 + +#endif /* STATE_BLT_XML */ diff --git a/drivers/gpu/drm/etnaviv/state_hi.xml.h b/drivers/gpu/drm/etnaviv/state_hi.xml.h index 60808daf7e8d..41d8da2b6f4f 100644 --- a/drivers/gpu/drm/etnaviv/state_hi.xml.h +++ b/drivers/gpu/drm/etnaviv/state_hi.xml.h @@ -1,4 +1,3 @@ -/* SPDX-License-Identifier: GPL-2.0 */ #ifndef STATE_HI_XML #define STATE_HI_XML @@ -9,10 +8,40 @@ http://0x04.net/cgit/index.cgi/rules-ng-ng git clone git://0x04.net/rules-ng-ng The rules-ng-ng source files this header was generated from are: -- state_hi.xml ( 25620 bytes, from 2016-08-19 22:07:37) -- common.xml ( 20583 bytes, from 2016-06-07 05:22:38) - -Copyright (C) 2016 +- state.xml ( 26087 bytes, from 2017-12-18 16:51:59) +- common.xml ( 35468 bytes, from 2018-01-22 13:48:54) +- common_3d.xml ( 14615 bytes, from 2017-12-18 16:51:59) +- state_hi.xml ( 30232 bytes, from 2018-02-15 15:48:01) +- copyright.xml ( 1597 bytes, from 2016-12-08 16:37:56) +- state_2d.xml ( 51552 bytes, from 2016-12-08 16:37:56) +- state_3d.xml ( 79992 bytes, from 2017-12-18 16:51:59) +- state_blt.xml ( 13405 bytes, from 2017-12-18 16:51:59) +- state_vg.xml ( 5975 bytes, from 2016-12-08 16:37:56) + +Copyright (C) 2012-2018 by the following authors: +- Wladimir J. van der Laan <laanwj@gmail.com> +- Christian Gmeiner <christian.gmeiner@gmail.com> +- Lucas Stach <l.stach@pengutronix.de> +- Russell King <rmk@arm.linux.org.uk> + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sub license, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice (including the +next paragraph) shall be included in all copies or substantial portions +of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL +THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +DEALINGS IN THE SOFTWARE. */ @@ -192,6 +221,9 @@ Copyright (C) 2016 #define VIVS_HI_CHIP_SPECS_3_GPU_CORE_COUNT__SHIFT 0 #define VIVS_HI_CHIP_SPECS_3_GPU_CORE_COUNT(x) (((x) << VIVS_HI_CHIP_SPECS_3_GPU_CORE_COUNT__SHIFT) & VIVS_HI_CHIP_SPECS_3_GPU_CORE_COUNT__MASK) +#define VIVS_HI_COMPRESSION_FLAGS 0x00000090 +#define VIVS_HI_COMPRESSION_FLAGS_DEC300 0x00000040 + #define VIVS_HI_CHIP_MINOR_FEATURE_4 0x00000094 #define VIVS_HI_CHIP_SPECS_4 0x0000009c @@ -203,6 +235,10 @@ Copyright (C) 2016 #define VIVS_HI_CHIP_PRODUCT_ID 0x000000a8 +#define VIVS_HI_BLT_INTR 0x000000d4 + +#define VIVS_HI_AUXBIT 0x000000ec + #define VIVS_PM 0x00000000 #define VIVS_PM_POWER_CONTROLS 0x00000100 @@ -239,6 +275,17 @@ Copyright (C) 2016 #define VIVS_PM_MODULE_STATUS_MODULE_CLOCK_GATED_TX 0x00000080 #define VIVS_PM_PULSE_EATER 0x0000010c +#define VIVS_PM_PULSE_EATER_DISABLE 0x00000001 +#define VIVS_PM_PULSE_EATER_DVFS_PERIOD__MASK 0x0000ff00 +#define VIVS_PM_PULSE_EATER_DVFS_PERIOD__SHIFT 8 +#define VIVS_PM_PULSE_EATER_DVFS_PERIOD(x) (((x) << VIVS_PM_PULSE_EATER_DVFS_PERIOD__SHIFT) & VIVS_PM_PULSE_EATER_DVFS_PERIOD__MASK) +#define VIVS_PM_PULSE_EATER_UNK16 0x00010000 +#define VIVS_PM_PULSE_EATER_UNK17 0x00020000 +#define VIVS_PM_PULSE_EATER_INTERNAL_DFS 0x00040000 +#define VIVS_PM_PULSE_EATER_UNK19 0x00080000 +#define VIVS_PM_PULSE_EATER_UNK20 0x00100000 +#define VIVS_PM_PULSE_EATER_UNK22 0x00400000 +#define VIVS_PM_PULSE_EATER_UNK23 0x00800000 #define VIVS_MMUv2 0x00000000 @@ -280,6 +327,68 @@ Copyright (C) 2016 #define VIVS_MMUv2_EXCEPTION_ADDR__ESIZE 0x00000004 #define VIVS_MMUv2_EXCEPTION_ADDR__LEN 0x00000004 +#define VIVS_MMUv2_PROFILE_BLT_READ 0x000001a4 + +#define VIVS_MMUv2_PTA_CONFIG 0x000001ac +#define VIVS_MMUv2_PTA_CONFIG_INDEX__MASK 0x0000ffff +#define VIVS_MMUv2_PTA_CONFIG_INDEX__SHIFT 0 +#define VIVS_MMUv2_PTA_CONFIG_INDEX(x) (((x) << VIVS_MMUv2_PTA_CONFIG_INDEX__SHIFT) & VIVS_MMUv2_PTA_CONFIG_INDEX__MASK) +#define VIVS_MMUv2_PTA_CONFIG_UNK16 0x00010000 + +#define VIVS_MMUv2_AXI_POLICY(i0) (0x000001c0 + 0x4*(i0)) +#define VIVS_MMUv2_AXI_POLICY__ESIZE 0x00000004 +#define VIVS_MMUv2_AXI_POLICY__LEN 0x00000008 + +#define VIVS_MMUv2_SEC_EXCEPTION_ADDR 0x00000380 + +#define VIVS_MMUv2_SEC_STATUS 0x00000384 +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION0__MASK 0x00000003 +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION0__SHIFT 0 +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION0(x) (((x) << VIVS_MMUv2_SEC_STATUS_EXCEPTION0__SHIFT) & VIVS_MMUv2_SEC_STATUS_EXCEPTION0__MASK) +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION1__MASK 0x00000030 +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION1__SHIFT 4 +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION1(x) (((x) << VIVS_MMUv2_SEC_STATUS_EXCEPTION1__SHIFT) & VIVS_MMUv2_SEC_STATUS_EXCEPTION1__MASK) +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION2__MASK 0x00000300 +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION2__SHIFT 8 +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION2(x) (((x) << VIVS_MMUv2_SEC_STATUS_EXCEPTION2__SHIFT) & VIVS_MMUv2_SEC_STATUS_EXCEPTION2__MASK) +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION3__MASK 0x00003000 +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION3__SHIFT 12 +#define VIVS_MMUv2_SEC_STATUS_EXCEPTION3(x) (((x) << VIVS_MMUv2_SEC_STATUS_EXCEPTION3__SHIFT) & VIVS_MMUv2_SEC_STATUS_EXCEPTION3__MASK) + +#define VIVS_MMUv2_SEC_CONTROL 0x00000388 +#define VIVS_MMUv2_SEC_CONTROL_ENABLE 0x00000001 + +#define VIVS_MMUv2_PTA_ADDRESS_LOW 0x0000038c + +#define VIVS_MMUv2_PTA_ADDRESS_HIGH 0x00000390 + +#define VIVS_MMUv2_PTA_CONTROL 0x00000394 +#define VIVS_MMUv2_PTA_CONTROL_ENABLE 0x00000001 + +#define VIVS_MMUv2_NONSEC_SAFE_ADDR_LOW 0x00000398 + +#define VIVS_MMUv2_SEC_SAFE_ADDR_LOW 0x0000039c + +#define VIVS_MMUv2_SAFE_ADDRESS_CONFIG 0x000003a0 +#define VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH__MASK 0x000000ff +#define VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH__SHIFT 0 +#define VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH(x) (((x) << VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH__SHIFT) & VIVS_MMUv2_SAFE_ADDRESS_CONFIG_NON_SEC_SAFE_ADDR_HIGH__MASK) +#define VIVS_MMUv2_SAFE_ADDRESS_CONFIG_UNK15 0x00008000 +#define VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH__MASK 0x00ff0000 +#define VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH__SHIFT 16 +#define VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH(x) (((x) << VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH__SHIFT) & VIVS_MMUv2_SAFE_ADDRESS_CONFIG_SEC_SAFE_ADDR_HIGH__MASK) +#define VIVS_MMUv2_SAFE_ADDRESS_CONFIG_UNK31 0x80000000 + +#define VIVS_MMUv2_SEC_COMMAND_CONTROL 0x000003a4 +#define VIVS_MMUv2_SEC_COMMAND_CONTROL_PREFETCH__MASK 0x0000ffff +#define VIVS_MMUv2_SEC_COMMAND_CONTROL_PREFETCH__SHIFT 0 +#define VIVS_MMUv2_SEC_COMMAND_CONTROL_PREFETCH(x) (((x) << VIVS_MMUv2_SEC_COMMAND_CONTROL_PREFETCH__SHIFT) & VIVS_MMUv2_SEC_COMMAND_CONTROL_PREFETCH__MASK) +#define VIVS_MMUv2_SEC_COMMAND_CONTROL_ENABLE 0x00010000 + +#define VIVS_MMUv2_AHB_CONTROL 0x000003a8 +#define VIVS_MMUv2_AHB_CONTROL_RESET 0x00000001 +#define VIVS_MMUv2_AHB_CONTROL_NONSEC_ACCESS 0x00000002 + #define VIVS_MC 0x00000000 #define VIVS_MC_MMU_FE_PAGE_TABLE 0x00000400 @@ -340,13 +449,13 @@ Copyright (C) 2016 #define VIVS_MC_PROFILE_HI_READ 0x0000046c #define VIVS_MC_PROFILE_CONFIG0 0x00000470 -#define VIVS_MC_PROFILE_CONFIG0_FE__MASK 0x0000000f +#define VIVS_MC_PROFILE_CONFIG0_FE__MASK 0x000000ff #define VIVS_MC_PROFILE_CONFIG0_FE__SHIFT 0 #define VIVS_MC_PROFILE_CONFIG0_FE_RESET 0x0000000f -#define VIVS_MC_PROFILE_CONFIG0_DE__MASK 0x00000f00 +#define VIVS_MC_PROFILE_CONFIG0_DE__MASK 0x0000ff00 #define VIVS_MC_PROFILE_CONFIG0_DE__SHIFT 8 #define VIVS_MC_PROFILE_CONFIG0_DE_RESET 0x00000f00 -#define VIVS_MC_PROFILE_CONFIG0_PE__MASK 0x000f0000 +#define VIVS_MC_PROFILE_CONFIG0_PE__MASK 0x00ff0000 #define VIVS_MC_PROFILE_CONFIG0_PE__SHIFT 16 #define VIVS_MC_PROFILE_CONFIG0_PE_PIXEL_COUNT_KILLED_BY_COLOR_PIPE 0x00000000 #define VIVS_MC_PROFILE_CONFIG0_PE_PIXEL_COUNT_KILLED_BY_DEPTH_PIPE 0x00010000 @@ -354,7 +463,7 @@ Copyright (C) 2016 #define VIVS_MC_PROFILE_CONFIG0_PE_PIXEL_COUNT_DRAWN_BY_DEPTH_PIPE 0x00030000 #define VIVS_MC_PROFILE_CONFIG0_PE_PIXELS_RENDERED_2D 0x000b0000 #define VIVS_MC_PROFILE_CONFIG0_PE_RESET 0x000f0000 -#define VIVS_MC_PROFILE_CONFIG0_SH__MASK 0x0f000000 +#define VIVS_MC_PROFILE_CONFIG0_SH__MASK 0xff000000 #define VIVS_MC_PROFILE_CONFIG0_SH__SHIFT 24 #define VIVS_MC_PROFILE_CONFIG0_SH_SHADER_CYCLES 0x04000000 #define VIVS_MC_PROFILE_CONFIG0_SH_PS_INST_COUNTER 0x07000000 @@ -368,7 +477,7 @@ Copyright (C) 2016 #define VIVS_MC_PROFILE_CONFIG0_SH_RESET 0x0f000000 #define VIVS_MC_PROFILE_CONFIG1 0x00000474 -#define VIVS_MC_PROFILE_CONFIG1_PA__MASK 0x0000000f +#define VIVS_MC_PROFILE_CONFIG1_PA__MASK 0x000000ff #define VIVS_MC_PROFILE_CONFIG1_PA__SHIFT 0 #define VIVS_MC_PROFILE_CONFIG1_PA_INPUT_VTX_COUNTER 0x00000003 #define VIVS_MC_PROFILE_CONFIG1_PA_INPUT_PRIM_COUNTER 0x00000004 @@ -377,12 +486,12 @@ Copyright (C) 2016 #define VIVS_MC_PROFILE_CONFIG1_PA_TRIVIAL_REJECTED_COUNTER 0x00000007 #define VIVS_MC_PROFILE_CONFIG1_PA_CULLED_COUNTER 0x00000008 #define VIVS_MC_PROFILE_CONFIG1_PA_RESET 0x0000000f -#define VIVS_MC_PROFILE_CONFIG1_SE__MASK 0x00000f00 +#define VIVS_MC_PROFILE_CONFIG1_SE__MASK 0x0000ff00 #define VIVS_MC_PROFILE_CONFIG1_SE__SHIFT 8 #define VIVS_MC_PROFILE_CONFIG1_SE_CULLED_TRIANGLE_COUNT 0x00000000 #define VIVS_MC_PROFILE_CONFIG1_SE_CULLED_LINES_COUNT 0x00000100 #define VIVS_MC_PROFILE_CONFIG1_SE_RESET 0x00000f00 -#define VIVS_MC_PROFILE_CONFIG1_RA__MASK 0x000f0000 +#define VIVS_MC_PROFILE_CONFIG1_RA__MASK 0x00ff0000 #define VIVS_MC_PROFILE_CONFIG1_RA__SHIFT 16 #define VIVS_MC_PROFILE_CONFIG1_RA_VALID_PIXEL_COUNT 0x00000000 #define VIVS_MC_PROFILE_CONFIG1_RA_TOTAL_QUAD_COUNT 0x00010000 @@ -392,7 +501,7 @@ Copyright (C) 2016 #define VIVS_MC_PROFILE_CONFIG1_RA_PREFETCH_CACHE_MISS_COUNTER 0x000a0000 #define VIVS_MC_PROFILE_CONFIG1_RA_CULLED_QUAD_COUNT 0x000b0000 #define VIVS_MC_PROFILE_CONFIG1_RA_RESET 0x000f0000 -#define VIVS_MC_PROFILE_CONFIG1_TX__MASK 0x0f000000 +#define VIVS_MC_PROFILE_CONFIG1_TX__MASK 0xff000000 #define VIVS_MC_PROFILE_CONFIG1_TX__SHIFT 24 #define VIVS_MC_PROFILE_CONFIG1_TX_TOTAL_BILINEAR_REQUESTS 0x00000000 #define VIVS_MC_PROFILE_CONFIG1_TX_TOTAL_TRILINEAR_REQUESTS 0x01000000 @@ -407,18 +516,21 @@ Copyright (C) 2016 #define VIVS_MC_PROFILE_CONFIG1_TX_RESET 0x0f000000 #define VIVS_MC_PROFILE_CONFIG2 0x00000478 -#define VIVS_MC_PROFILE_CONFIG2_MC__MASK 0x0000000f +#define VIVS_MC_PROFILE_CONFIG2_MC__MASK 0x000000ff #define VIVS_MC_PROFILE_CONFIG2_MC__SHIFT 0 #define VIVS_MC_PROFILE_CONFIG2_MC_TOTAL_READ_REQ_8B_FROM_PIPELINE 0x00000001 #define VIVS_MC_PROFILE_CONFIG2_MC_TOTAL_READ_REQ_8B_FROM_IP 0x00000002 #define VIVS_MC_PROFILE_CONFIG2_MC_TOTAL_WRITE_REQ_8B_FROM_PIPELINE 0x00000003 #define VIVS_MC_PROFILE_CONFIG2_MC_RESET 0x0000000f -#define VIVS_MC_PROFILE_CONFIG2_HI__MASK 0x00000f00 +#define VIVS_MC_PROFILE_CONFIG2_HI__MASK 0x0000ff00 #define VIVS_MC_PROFILE_CONFIG2_HI__SHIFT 8 #define VIVS_MC_PROFILE_CONFIG2_HI_AXI_CYCLES_READ_REQUEST_STALLED 0x00000000 #define VIVS_MC_PROFILE_CONFIG2_HI_AXI_CYCLES_WRITE_REQUEST_STALLED 0x00000100 #define VIVS_MC_PROFILE_CONFIG2_HI_AXI_CYCLES_WRITE_DATA_STALLED 0x00000200 #define VIVS_MC_PROFILE_CONFIG2_HI_RESET 0x00000f00 +#define VIVS_MC_PROFILE_CONFIG2_BLT__MASK 0xff000000 +#define VIVS_MC_PROFILE_CONFIG2_BLT__SHIFT 24 +#define VIVS_MC_PROFILE_CONFIG2_BLT_UNK0 0x00000000 #define VIVS_MC_PROFILE_CONFIG3 0x0000047c @@ -432,7 +544,13 @@ Copyright (C) 2016 #define VIVS_MC_START_COMPOSITION 0x00000554 -#define VIVS_MC_128B_MERGE 0x00000558 +#define VIVS_MC_FLAGS 0x00000558 +#define VIVS_MC_FLAGS_128B_MERGE 0x00000001 +#define VIVS_MC_FLAGS_TPCV11_COMPRESSION 0x08000000 + +#define VIVS_MC_L2_CACHE_CONFIG 0x0000055c + +#define VIVS_MC_PROFILE_L2_READ 0x00000564 #endif /* STATE_HI_XML */ |